From 4b9607e86e27fbfe460c94e4979ddae654e78b6a Mon Sep 17 00:00:00 2001 From: Vasuki Narayana Date: Fri, 29 Apr 2022 14:15:06 +0530 Subject: [PATCH 1/3] Adding emit_invalid_record_to_error false to parsers to suppress unwated pattern match failure errors Adding emit_invalid_record_to_error false to parsers to suppress unwated pattern match failure errors --- .gitignore | 3 +++ fluent.conf.rt | 7 ++++--- fluent.conf.xray | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ae7b59a --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ + +.DS_Store +.ruby-version diff --git a/fluent.conf.rt b/fluent.conf.rt index 9a010f6..93af92b 100644 --- a/fluent.conf.rt +++ b/fluent.conf.rt @@ -65,6 +65,7 @@ time_key timestamp time_format %Y-%m-%dT%H:%M:%S.%LZ + emit_invalid_record_to_error false @type tail @@ -189,7 +190,7 @@ @type exec tag jfrog.callhome - command "curl --request GET 'http://localhost:8081/artifactory/api/system/version' -H 'Authorization: Bearer '" + command "curl --request GET 'JPD_URL/artifactory/api/system/version' -H 'Authorization: Bearer JFROG_ADMIN_TOKEN'" run_interval 1d @type json @@ -207,10 +208,10 @@ @type http - endpoint http://localhost:8081/artifactory/api/system/usage + endpoint JPD_URL/artifactory/api/system/usage open_timeout 5 content_type application/json - headers {"Authorization":"Bearer "} + headers {"Authorization":"Bearer JFROG_ADMIN_TOKEN"} @type json diff --git a/fluent.conf.xray b/fluent.conf.xray index dcc8678..3942ad9 100644 --- a/fluent.conf.xray +++ b/fluent.conf.xray @@ -103,6 +103,7 @@ time_key timestamp time_format %Y-%m-%dT%H:%M:%S.%LZ + emit_invalid_record_to_error false @type record_transformer From ccd1c587d239af8610cc4ce818e2e3fa26ac6851 Mon Sep 17 00:00:00 2001 From: Vasuki Narayana Date: Tue, 3 May 2022 19:12:12 +0530 Subject: [PATCH 2/3] Adding support for JFrog Platform Charts Adding support for JFrog Platform Charts --- README.md | 106 ++++++++++++++++-- ...jfrog-platform-values-without-datadog.yaml | 17 +++ helm/jfrog-platform-values.yaml | 77 +++++++++++++ 3 files changed, 192 insertions(+), 8 deletions(-) create mode 100644 helm/jfrog-platform-values-without-datadog.yaml create mode 100644 helm/jfrog-platform-values.yaml diff --git a/README.md b/README.md index eee3ab1..87a61b7 100644 --- a/README.md +++ b/README.md @@ -139,11 +139,12 @@ Configure `fluent.conf.*` according to the instructions mentioned in [Fluentd Co Recommended installation for Kubernetes is to utilize the helm chart with the associated values.yaml in this repo. -| Product | Example Values File | -|---------|-------------| -| Artifactory | helm/artifactory-values.yaml | +| Product | Example Values File | +|----------------|-------------| +| Platform | helm/jfrog-platform-values.yaml | +| Artifactory | helm/artifactory-values.yaml | | Artifactory HA | helm/artifactory-ha-values.yaml | -| Xray | helm/xray-values.yaml | +| Xray | helm/xray-values.yaml | Update the values.yaml associated to the product you want to deploy with your Datadog settings. @@ -151,11 +152,100 @@ Then deploy the helm chart as described below: Add JFrog Helm repository: -```text +```kubernetes helm helm repo add jfrog https://charts.jfrog.io helm repo update ``` +JFrog Platform ⎈: + +```textmate +!!!Important Note!!!: Platform Chart Deployment shown here is for reference purpose only, +Kindly apply these charts only after reviewing the options and make necessary changes that suits your deployment +``` + +Pre-requisite for Observability integration with JFrog Platform charts + +First, install the JFrog Platform chart by configuring intended replicaCount for Artifactory, Xray and enable or disable the solutions + +Refer the sample yaml for reference, download [here](https://github.com/jfrog/log-analytics-datadog/blob/master/helm/jfrog-platform-values-without-datadog.yaml) + +```yaml + +installerInfo: '{ "productId": "Helm_datadog_artifactory/{{ .Chart.Version }}", "features": [ { "featureId": "ArtifactoryVersion/{{ default .Chart.AppVersion .Values.artifactory.image.version }}" }, { "featureId": "{{ if .Values.postgresql.enabled }}postgresql{{ else }}{{ .Values.database.type }}{{ end }}/0.0.0" }, { "featureId": "Platform/{{ default "kubernetes" .Values.installer.platform }}" }, { "featureId": "Channel/Helm_datadog_artifactory" } ] }' +artifactory: + artifactory: + openMetrics: + enabled: true +xray: + enabled: true + replicaCount: 1 +insight: + enabled: false +distribution: + enabled: false +pipelines: + enabled: false +rabbitmq: + enabled: true +redis: + enabled: false + +``` +To install the JFrog Platform with the above said configurations run the following command, (note the namespaces and adjust as needed by your deployment requirement) + +```kubernetes helm +helm upgrade --install jfrog-platform --namespace jfrog-platform jfrog/jfrog-platform -f jfrog-platform-values-without-datadog.yaml +``` + +Once the platform is accessible, login to the platform and perform the setup as directed on the UI. + +Second, when your JFrog Platform is ready and accessible, the following should be noted + +1. Access Token - click [here](https://www.jfrog.com/confluence/display/JFROG/Access+Tokens#AccessTokens-GeneratingScopedTokens) to know how to generate a admin scoped access token +2. API Key - click [here](https://www.jfrog.com/confluence/display/RTF6X/Updating+Your+Profile#UpdatingYourProfile-APIKey) to generate an API Key with profile update process +3. User - admin + Refer [Fluentd Configuration for Datadog](#fluentd-configuration-for-datadog) section for configuring the below parameters +4. Datadog API Key - click [here](https://docs.datadoghq.com/account_management/api-app-keys/), to get the API Key which should be used to send data to Datadog + +Once the values are noted, download the file to apply the JFrog Platform Upgrade for Datadog from [here](https://github.com/jfrog/log-analytics-datadog/blob/master/helm/jfrog-platform-values.yaml) + +Replace the respective values for the following in the global segment of chart values, review the chart values and apply them accordingly + +```yaml +global: + datadog: + apikey: datadog_api_key + jfrog: + observability: + metrics: + jpd_url: http://localhost:8082 + jpd_url_nginx: http://jfrog-platform-artifactory-nginx + username: jfrog_user + apikey: jfrog_api_key + token: jfrog_token + branch: master +``` + +Once the values are replaced, apply the upgrade as mentioned + +1. Get the JFrog Platform Postgres Password and store it to a variable +```shell +export POSTGRES_PASSWORD=kubectl get secret -n jfrog-platform jfrog-platform-postgresql -o jsonpath="{.data.postgresql-password}" | base64 --decode +``` +2. Using the password obtained, run the following upgrade command +```kubernetes helm +helm upgrade jfrog-platform --namespace jfrog-platform jfrog/jfrog-platform --set databaseUpgradeReady=true --set postgresql.postgresqlPassword=$POSTGRES_PASSWORD -f jfrog-platform-values.yaml +``` +3. For getting metrics data kindly follow the documentation on Datadog agent based setup [here](https://github.com/jfrog/metrics/blob/main/datadog/README.md) + +```textmate +!!!Important Note!!!: Platform Chart Deployment shown here is for reference purpose only, +Kindly apply these charts only after reviewing the options and make necessary changes that suits your deployment +``` + + + Replace placeholders with your ``masterKey`` and ``joinKey``. To generate each of them, use the command ``openssl rand -hex 32`` @@ -163,7 +253,7 @@ Artifactory ⎈: Replace the `datadog_api_key` at the end of the yaml file with apiKey from [Datadog](https://docs.datadoghq.com/account_management/api-app-keys/) and then run the following helm command: -```text +```kubernetes helm helm upgrade --install artifactory jfrog/artifactory \ --set artifactory.masterKey=FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF \ --set artifactory.joinKey=EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE \ @@ -183,7 +273,7 @@ Note: Replace placeholders with your ``masterKey`` and ``joinKey``. To generate Replace the `datadog_api_key` at the end of the yaml file with apiKey from [Datadog](https://docs.datadoghq.com/account_management/api-app-keys/) and then run the following helm command -```text +```kubernetes helm helm upgrade --install artifactory-ha jfrog/artifactory-ha \ --set artifactory.masterKey=FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF \ --set artifactory.joinKey=EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE \ @@ -204,7 +294,7 @@ Replace `jfrog_jpd_url` in `jfrog.siem.jpdUrl` with Artifactory JPD URL of the f Use the same `joinKey` as you used in Artifactory installation to allow Xray node to successfully connect to Artifactory. -```text +```kubernetes helm helm upgrade --install xray jfrog/xray --set xray.jfrogUrl=http://my-artifactory-nginx-url \ --set xray.masterKey=FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF \ --set xray.joinKey=EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE \ diff --git a/helm/jfrog-platform-values-without-datadog.yaml b/helm/jfrog-platform-values-without-datadog.yaml new file mode 100644 index 0000000..d21b6f8 --- /dev/null +++ b/helm/jfrog-platform-values-without-datadog.yaml @@ -0,0 +1,17 @@ +installerInfo: '{ "productId": "Helm_datadog_artifactory/{{ .Chart.Version }}", "features": [ { "featureId": "ArtifactoryVersion/{{ default .Chart.AppVersion .Values.artifactory.image.version }}" }, { "featureId": "{{ if .Values.postgresql.enabled }}postgresql{{ else }}{{ .Values.database.type }}{{ end }}/0.0.0" }, { "featureId": "Platform/{{ default "kubernetes" .Values.installer.platform }}" }, { "featureId": "Channel/Helm_datadog_artifactory" } ] }' +artifactory: + artifactory: + openMetrics: + enabled: true +xray: + enabled: true +insight: + enabled: false +distribution: + enabled: false +pipelines: + enabled: false +rabbitmq: + enabled: true +redis: + enabled: false diff --git a/helm/jfrog-platform-values.yaml b/helm/jfrog-platform-values.yaml new file mode 100644 index 0000000..c97e11c --- /dev/null +++ b/helm/jfrog-platform-values.yaml @@ -0,0 +1,77 @@ +global: + datadog: + apikey: datadog_api_key + jfrog: + observability: + metrics: + jpd_url: http://localhost:8082 + jpd_url_nginx: http://jfrog-platform-artifactory-nginx + username: jfrog_user + apikey: jfrog_api_key + token: jfrog_token + branch: master + +installerInfo: '{ "productId": "Helm_datadog_artifactory/{{ .Chart.Version }}", "features": [ { "featureId": "ArtifactoryVersion/{{ default .Chart.AppVersion .Values.artifactory.image.version }}" }, { "featureId": "{{ if .Values.postgresql.enabled }}postgresql{{ else }}{{ .Values.database.type }}{{ end }}/0.0.0" }, { "featureId": "Platform/{{ default "kubernetes" .Values.installer.platform }}" }, { "featureId": "Channel/Helm_datadog_artifactory" } ] }' +artifactory: + artifactory: + openMetrics: + enabled: true + customSidecarContainers: | + - name: "artifactory-fluentd-sidecar" + image: "partnership-public-images.jfrog.io/fluentd/fluentd:1.18.0" + imagePullPolicy: "IfNotPresent" + command: + - 'sh' + - '-c' + - > + curl https://raw.githubusercontent.com/jfrog/log-analytics-datadog/{{ .Values.global.jfrog.observability.branch }}/fluent.conf.rt -o fluentd.conf; + cp fluentd.conf fluent.tmp.conf; + sed -i -e 's|JPD_URL|{{ .Values.global.jfrog.observability.metrics.jpd_url }}|g' \ + -e 's/JFROG_ADMIN_TOKEN/{{ .Values.global.jfrog.observability.metrics.token }}/' \ + -e 's/API_KEY/{{ .Values.global.datadog.apikey }}/' fluent.tmp.conf; + mv fluent.tmp.conf fluentd.conf; + fluentd --suppress-config-dump -c fluentd.conf + volumeMounts: + - mountPath: "{{ .Values.artifactory.persistence.mountPath }}" + name: artifactory-volume + env: + - name: JF_PRODUCT_DATA_INTERNAL + value: {{ .Values.artifactory.persistence.mountPath }} +xray: + enabled: true + common: + customSidecarContainers: | + - name: "xray-fluentd-sidecar" + image: "partnership-public-images.jfrog.io/fluentd/fluentd:1.18.0" + imagePullPolicy: "IfNotPresent" + command: + - 'sh' + - '-c' + - > + curl https://raw.githubusercontent.com/jfrog/log-analytics-datadog/{{ .Values.global.jfrog.observability.branch }}/fluent.conf.xray -o fluentd.conf; + cp fluentd.conf fluent.tmp.conf; + sed -i -e 's/DATADOG_API_KEY/{{ .Values.global.datadog.apikey }}/' \ + -e 's|JPD_URL|{{ .Values.global.jfrog.observability.metrics.jpd_url }}|g' \ + -e 's/ADMIN_USERNAME/{{ .Values.global.jfrog.observability.metrics.username }}/' \ + -e 's/JFROG_ADMIN_TOKEN/{{ .Values.global.jfrog.observability.metrics.token }}/' \ + -e 's/JFROG_API_KEY/{{ .Values.global.jfrog.observability.metrics.apikey }}/' fluent.tmp.conf; + [ `hostname` != "jfrog-platform-xray-0" ] && sed -i -e 's/jfrog_siem/dummy/g' fluent.tmp.conf; + [ `hostname` != "jfrog-platform-xray-0" ] && sed -i -e 's/tag jfrog.xray.siem.vulnerabilities/tag ignored\n suspend true/g' fluent.tmp.conf; + mv fluent.tmp.conf fluentd.conf; + fluentd --suppress-config-dump -c fluentd.conf + volumeMounts: + - mountPath: "{{ .Values.xray.persistence.mountPath }}" + name: data-volume + env: + - name: JF_PRODUCT_DATA_INTERNAL + value: {{ .Values.xray.persistence.mountPath }} +insight: + enabled: false +distribution: + enabled: false +pipelines: + enabled: false +rabbitmq: + enabled: true +redis: + enabled: false From 00f4e7804150db1016914e9a79f91834d819eb60 Mon Sep 17 00:00:00 2001 From: Vasuki Narayana Date: Tue, 3 May 2022 19:15:17 +0530 Subject: [PATCH 3/3] Update to Readme Update to Readme --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 87a61b7..33dea89 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,9 @@ Second, when your JFrog Platform is ready and accessible, the following should b 1. Access Token - click [here](https://www.jfrog.com/confluence/display/JFROG/Access+Tokens#AccessTokens-GeneratingScopedTokens) to know how to generate a admin scoped access token 2. API Key - click [here](https://www.jfrog.com/confluence/display/RTF6X/Updating+Your+Profile#UpdatingYourProfile-APIKey) to generate an API Key with profile update process 3. User - admin - Refer [Fluentd Configuration for Datadog](#fluentd-configuration-for-datadog) section for configuring the below parameters + +Refer [Fluentd Configuration for Datadog](#fluentd-configuration-for-datadog) section for configuring the below parameters + 4. Datadog API Key - click [here](https://docs.datadoghq.com/account_management/api-app-keys/), to get the API Key which should be used to send data to Datadog Once the values are noted, download the file to apply the JFrog Platform Upgrade for Datadog from [here](https://github.com/jfrog/log-analytics-datadog/blob/master/helm/jfrog-platform-values.yaml)