From a2a39f9bf7c89046ba3478f90a66059699f140d9 Mon Sep 17 00:00:00 2001 From: Charlie Le Date: Wed, 26 Jun 2024 20:59:28 -0700 Subject: [PATCH] Fix getting started guide - Update to Cortex v1.17.1 - Configure Seaweedfs auth (uploads were failing otherwise) - Add some experiments to play with Cortex - Automate Grafana datasource creation - Remove Grafana auth to speed up getting started - Set scrape interval to 15s in K8s for more data Signed-off-by: Charlie Le --- docs/getting-started/.env | 2 +- docs/getting-started/_index.md | 87 ++++++++++++-------- docs/getting-started/cortex-config.yaml | 8 +- docs/getting-started/cortex-values.yaml | 6 +- docs/getting-started/docker-compose.yaml | 22 +++-- docs/getting-started/grafana-datasource.yaml | 11 +++ docs/getting-started/grafana-values.yaml | 26 +++--- docs/getting-started/prometheus-values.yaml | 2 +- docs/getting-started/seaweedfs-config.json | 16 ++++ docs/getting-started/seaweedfs.yaml | 32 +++++++ 10 files changed, 154 insertions(+), 58 deletions(-) create mode 100644 docs/getting-started/grafana-datasource.yaml create mode 100644 docs/getting-started/seaweedfs-config.json diff --git a/docs/getting-started/.env b/docs/getting-started/.env index d96073c23f..58aa8c3d91 100644 --- a/docs/getting-started/.env +++ b/docs/getting-started/.env @@ -1,4 +1,4 @@ -CORTEX_VERSION=v1.17.0 +CORTEX_VERSION=v1.17.1 GRAFANA_VERSION=10.4.2 PROMETHEUS_VERSION=v2.51.2 SEAWEEDFS_VERSION=3.67 \ No newline at end of file diff --git a/docs/getting-started/_index.md b/docs/getting-started/_index.md index 68cb830d00..54c8ede5c6 100644 --- a/docs/getting-started/_index.md +++ b/docs/getting-started/_index.md @@ -57,25 +57,27 @@ how this is configured. ```sh # Create a bucket in SeaweedFS -curl -X PUT http://localhost:8333/cortex-bucket +curl --aws-sigv4 "aws:amz:local:seaweedfs" --user "any:any" -X PUT http://localhost:8333/cortex-bucket ``` -#### Configure Grafana +#### Explore -1. Log into the Grafana instance at [http://localhost:3000](http://localhost:3000) - * login credentials are `username: admin` and `password: admin` - * There may be an additional screen on setting a new password. This can be skipped and is optional -1. Navigate to the `Data Sources` page - * Look for a gear icon on the left sidebar and select `Data Sources` -1. Add a new Prometheus Data Source - * Use `http://cortex:9009/api/prom` as the URL - * Click `Save & Test` -1. Go to `Metrics Explore` to query metrics - * Look for a compass icon on the left sidebar - * Click `Metrics` for a dropdown list of all the available metrics +Grafana is configured to use Cortex as a data source. You can explore the data source in Grafana and query metrics. For example, this [explore](http://localhost:3000/explore?schemaVersion=1&panes=%7B%22au0%22:%7B%22datasource%22:%22P6693426190CB2316%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22rate%28prometheus_remote_storage_samples_total%5B$__rate_interval%5D%29%22,%22range%22:true,%22instant%22:true,%22datasource%22:%7B%22type%22:%22prometheus%22,%22uid%22:%22P6693426190CB2316%22%7D,%22editorMode%22:%22builder%22,%22legendFormat%22:%22__auto%22,%22useBackend%22:false,%22disableTextWrap%22:false,%22fullMetaSearch%22:false,%22includeNullMetadata%22:false%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D%7D&orgId=1) page is showing the rate of samples being sent to Cortex. If everything is working correctly, then the metrics seen in Grafana were successfully sent from Prometheus to Cortex -via remote_write! +via `remote_write`! + +Other things to explore: + +- [Cortex](http://localhost:9009) - Administrative interface for Cortex + - Try shutting down the [ingester](http://localhost:9009/ingester/shutdown) and see how it affects metric ingestion. + - Restart Cortex to bring the ingester back online, and see how Prometheus catches up. + - Does it affect the querying of metrics in Grafana? +- [Prometheus](http://localhost:9090) - Prometheus instance that is sending metrics to Cortex + - Try querying the metrics in Prometheus. + - Are they the same as what you see in Cortex? +- [Grafana](http://localhost:3000) - Grafana instance that is visualizing the metrics. + - Try creating a new dashboard and adding a new panel with a query to Cortex. ### Clean up @@ -140,7 +142,7 @@ $ kubectl -n cortex port-forward svc/seaweedfs 8333 ```shell # Create a bucket -$ curl -X PUT http://localhost:8333/cortex-bucket +$ curl --aws-sigv4 "aws:amz:local:seaweedfs" --user "any:any" -X PUT http://localhost:8333/cortex-bucket ``` #### Setup Cortex @@ -148,47 +150,66 @@ $ curl -X PUT http://localhost:8333/cortex-bucket ```sh # Deploy Cortex using the provided values file which configures # - blocks storage to use the seaweedfs service -$ helm install --version=2.3.0 --namespace cortex cortex cortex-helm/cortex -f cortex-values.yaml +$ helm upgrade --install --version=2.3.0 --namespace cortex cortex cortex-helm/cortex -f cortex-values.yaml ``` #### Setup Prometheus ```sh # Deploy Prometheus to scrape metrics in the cluster and send them, via remote_write, to Cortex. -$ helm install --version=25.20.1 --namespace cortex prometheus prometheus-community/prometheus -f prometheus-values.yaml +$ helm upgrade --install --version=25.20.1 --namespace cortex prometheus prometheus-community/prometheus -f prometheus-values.yaml ``` #### Setup Grafana ```sh # Deploy Grafana to visualize the metrics that were sent to Cortex. -$ helm install --version=7.3.9 --namespace cortex grafana grafana/grafana +$ helm upgrade --install --version=7.3.9 --namespace cortex grafana grafana/grafana -f grafana-values.yaml ``` -#### Configure Grafana - -```sh -# Get your 'admin' user password -kubectl get secret --namespace cortex grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo -``` +#### Explore ```sh # Port-forward to Grafana to visualize -kubectl --namespace cortex port-forward deploy/cortex 3000 +kubectl --namespace cortex port-forward deploy/grafana 3000 ``` -1. Log into the Grafana instance at [http://localhost:3000](http://localhost:3000) -1. Use the username `admin` and the password from the Kubernetes secret -1. Navigate to the [Data Sources](http://localhost:3000/connections/datasources) page -1. Add a new Prometheus Data Source -1. Use `http://cortex-nginx/api/prom` as the URL -1. Click `Save & Test` -1. Go to [Explore](http://localhost:3000/explore) to query metrics -1. Click `Metrics` for a dropdown list of all the available metrics +Grafana is configured to use Cortex as a data source. You can explore the data source in Grafana and query metrics. For example, this [explore](http://localhost:3000/explore?schemaVersion=1&panes=%7B%22au0%22:%7B%22datasource%22:%22P6693426190CB2316%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22rate%28prometheus_remote_storage_samples_total%5B$__rate_interval%5D%29%22,%22range%22:true,%22instant%22:true,%22datasource%22:%7B%22type%22:%22prometheus%22,%22uid%22:%22P6693426190CB2316%22%7D,%22editorMode%22:%22builder%22,%22legendFormat%22:%22__auto%22,%22useBackend%22:false,%22disableTextWrap%22:false,%22fullMetaSearch%22:false,%22includeNullMetadata%22:false%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D%7D&orgId=1) page is showing the rate of samples being sent to Cortex. + If everything is working correctly, then the metrics seen in Grafana were successfully sent from Prometheus to Cortex via remote_write! +Other things to explore: + +```sh +# Port forward to the ingester to see the administrative interface for Cortex: +$ kubectl --namespace cortex port-forward deploy/cortex-ingester 8080 +``` + +- [Cortex Ingester](http://localhost:8080) + - Try shutting down the [ingester](http://localhost:8080/ingester/shutdown) and see how it affects metric ingestion. + - Restart ingester pod to bring the ingester back online, and see if Prometheus affected. + - Does it affect the querying of metrics in Grafana? How many ingesters must be offline before it affects querying? + + +```sh +# Port forward to Prometheus to see the metrics that are being scraped: +$ kubectl --namespace cortex port-forward deploy/prometheus-server 9090 +``` + +- [Prometheus](http://localhost:9090) - Prometheus instance that is sending metrics to Cortex + - Try querying the metrics in Prometheus. + - Are they the same as what you see in Cortex? + +```sh +# Port forward to Prometheus to see the metrics that are being scraped: +$ kubectl --namespace cortex port-forward deploy/grafana 3000 +``` + +- [Grafana](http://localhost:3000) - Grafana instance that is visualizing the metrics. + - Try creating a new dashboard and adding a new panel with a query to Cortex. + ### Clean up ```sh diff --git a/docs/getting-started/cortex-config.yaml b/docs/getting-started/cortex-config.yaml index f86f1c71d0..d11b626ded 100644 --- a/docs/getting-started/cortex-config.yaml +++ b/docs/getting-started/cortex-config.yaml @@ -5,7 +5,7 @@ target: all,compactor -# Disable the requirement that every request to Cortex has a +# Disable the requirement that every request to Cortex has an # X-Scope-OrgID header. `fake` will be substituted in instead. auth_enabled: false @@ -49,10 +49,10 @@ ingester: blocks_storage: s3: &s3 endpoint: seaweedfs:8333 - region: baz + region: local bucket_name: cortex-bucket - access_key_id: foo - secret_access_key: bar + access_key_id: any + secret_access_key: any insecure: true tsdb: dir: /data/tsdb diff --git a/docs/getting-started/cortex-values.yaml b/docs/getting-started/cortex-values.yaml index 477f4a8eef..1be7862b1e 100644 --- a/docs/getting-started/cortex-values.yaml +++ b/docs/getting-started/cortex-values.yaml @@ -81,10 +81,10 @@ config: blocks_storage: s3: &s3 endpoint: seaweedfs.cortex.svc.cluster.local:8333 - region: baz + region: local bucket_name: cortex-bucket - access_key_id: foo - secret_access_key: bar + access_key_id: any + secret_access_key: any insecure: true tsdb: dir: /data/tsdb diff --git a/docs/getting-started/docker-compose.yaml b/docs/getting-started/docker-compose.yaml index 8af2bdf694..f78295a4fc 100644 --- a/docs/getting-started/docker-compose.yaml +++ b/docs/getting-started/docker-compose.yaml @@ -10,8 +10,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -version: "3.8" - services: cortex: image: quay.io/cortexproject/cortex:${CORTEX_VERSION} @@ -20,16 +18,25 @@ services: volumes: - ./cortex-config.yaml:/config/cortex-config.yaml:ro ports: - - 9009:9009 + - "9009:9009" healthcheck: test: wget -qO- http://localhost:9009/ready interval: 10s timeout: 10s retries: 3 + restart: on-failure grafana: image: grafana/grafana:${GRAFANA_VERSION} + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_USERS_DEFAULT_THEME=light + - GF_LOG_MODE=console + - GF_LOG_LEVEL=critical + volumes: + - ./grafana-datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yaml:ro ports: - - 3000:3000 + - "3000:3000" prometheus: image: prom/prometheus:${PROMETHEUS_VERSION} command: @@ -37,14 +44,17 @@ services: volumes: - ./prometheus-config.yaml:/config/prometheus-config.yaml:ro ports: - - 9090:9090 + - "9090:9090" seaweedfs: image: chrislusf/seaweedfs:${SEAWEEDFS_VERSION} command: - server - -s3 + - -s3.config=/workspace/seaweedfs-config.json ports: - - 8333:8333 + - "8333:8333" + volumes: + - ./seaweedfs-config.json:/workspace/seaweedfs-config.json:ro healthcheck: test: wget -qO- http://localhost:8333/status interval: 10s diff --git a/docs/getting-started/grafana-datasource.yaml b/docs/getting-started/grafana-datasource.yaml new file mode 100644 index 0000000000..62a39c88db --- /dev/null +++ b/docs/getting-started/grafana-datasource.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 + +datasources: + - name: Cortex + type: prometheus + access: proxy + orgId: 1 + url: http://cortex:9009/api/prom + version: 1 + editable: true + isDefault: true diff --git a/docs/getting-started/grafana-values.yaml b/docs/getting-started/grafana-values.yaml index 0e6ba7f361..d612cb27d8 100644 --- a/docs/getting-started/grafana-values.yaml +++ b/docs/getting-started/grafana-values.yaml @@ -465,7 +465,12 @@ admin: ## 6. uncomment the annotation section in the serviceAccount: above ## make sure to replace arn:aws:iam::123456789000:role/iam-role-name-here with your role arn -env: {} +env: + GF_AUTH_ANONYMOUS_ENABLED: true + GF_AUTH_ANONYMOUS_ORG_ROLE: Admin + GF_USERS_DEFAULT_THEME: light + GF_LOG_MODE: console + GF_LOG_LEVEL: critical ## "valueFrom" environment variable references that will be added to deployment pods. Name is templated. ## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core @@ -590,15 +595,16 @@ plugins: [] ## Configure grafana datasources ## ref: http://docs.grafana.org/administration/provisioning/#datasources ## -datasources: {} -# datasources.yaml: -# apiVersion: 1 -# datasources: -# - name: Prometheus -# type: prometheus -# url: http://prometheus-prometheus-server -# access: proxy -# isDefault: true +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + - name: Cortex + type: prometheus + url: http://cortex-nginx/api/prom + access: proxy + isDefault: true + editable: true # - name: CloudWatch # type: cloudwatch # access: proxy diff --git a/docs/getting-started/prometheus-values.yaml b/docs/getting-started/prometheus-values.yaml index 653cc093bd..c7ca147ad7 100644 --- a/docs/getting-started/prometheus-values.yaml +++ b/docs/getting-started/prometheus-values.yaml @@ -238,7 +238,7 @@ server: global: ## How frequently to scrape targets by default ## - scrape_interval: 1m + scrape_interval: 15s ## How long until a scrape request times out ## scrape_timeout: 10s diff --git a/docs/getting-started/seaweedfs-config.json b/docs/getting-started/seaweedfs-config.json new file mode 100644 index 0000000000..75b0e4b955 --- /dev/null +++ b/docs/getting-started/seaweedfs-config.json @@ -0,0 +1,16 @@ +{ + "identities": [ + { + "name": "cortex", + "credentials": [ + { + "accessKey": "any", + "secretKey": "any" + } + ], + "actions": [ + "Admin" + ] + } + ] +} \ No newline at end of file diff --git a/docs/getting-started/seaweedfs.yaml b/docs/getting-started/seaweedfs.yaml index 7461e213fd..5ca4f24ddb 100644 --- a/docs/getting-started/seaweedfs.yaml +++ b/docs/getting-started/seaweedfs.yaml @@ -1,3 +1,27 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + creationTimestamp: null + name: seaweedfs-config +data: + seaweedfs-config.json: |- + { + "identities": [ + { + "name": "cortex", + "credentials": [ + { + "accessKey": "any", + "secretKey": "any" + } + ], + "actions": [ + "Admin" + ] + } + ] + } +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -22,8 +46,16 @@ spec: args: - "server" - "-s3" + - "-s3.config=/workspace/seaweedfs-config.json" ports: - containerPort: 8333 + volumeMounts: + - name: seaweedfs-config + mountPath: /workspace + volumes: + - name: seaweedfs-config + configMap: + name: seaweedfs-config restartPolicy: Always --- apiVersion: v1