From c8a7ad6651047897d85224c96edc871dda0c42c3 Mon Sep 17 00:00:00 2001
From: "tomas.panik" <tomas.panik@solarwinds.com>
Date: Fri, 1 Jul 2022 10:55:42 +0200
Subject: [PATCH 1/4] NH-9834: Adding collection of container logs

---
 build/docker/Dockerfile                       |   1 +
 build/logs-otel-collector-config.yaml         | 169 ++++++++++++++++++
 ...nighthawk-swi-opentelemetry-collector.yaml |   3 +-
 deploy/k8s/manifest.yaml                      |  89 ++++++++-
 tests/deploy/base/kustomization.yaml          |   1 +
 .../base/patch-logs-apply-env-variables.yaml  |  22 +++
 6 files changed, 283 insertions(+), 2 deletions(-)
 create mode 100644 build/logs-otel-collector-config.yaml
 create mode 100644 tests/deploy/base/patch-logs-apply-env-variables.yaml

diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile
index dba7f374..0182c328 100644
--- a/build/docker/Dockerfile
+++ b/build/docker/Dockerfile
@@ -25,6 +25,7 @@ ENV SCRAPE_INTERVAL="60s"
 ENV OTEL_ENVOY_ADDRESS_TLS_INSECURE="false"
 
 COPY /build/otel-collector-config.yaml /opt/default-config.yaml
+COPY /build/logs-otel-collector-config.yaml /opt/log-config.yaml
 
 ENTRYPOINT ["/swi-otelcol"]
 CMD ["--config=/opt/default-config.yaml"]
diff --git a/build/logs-otel-collector-config.yaml b/build/logs-otel-collector-config.yaml
new file mode 100644
index 00000000..a9169556
--- /dev/null
+++ b/build/logs-otel-collector-config.yaml
@@ -0,0 +1,169 @@
+exporters:
+  otlp:
+    endpoint: ${OTEL_ENVOY_ADDRESS}
+    tls:
+      insecure: ${OTEL_ENVOY_ADDRESS_TLS_INSECURE}
+    headers:
+      "Authorization": "Bearer ${SOLARWINDS_API_TOKEN}"
+extensions:
+  health_check: {}
+
+processors:
+  groupbyattrs/all:
+    keys:
+      - k8s.container.name
+      - k8s.namespace.name
+      - k8s.pod.name
+      - k8s.pod.uid
+  
+  resource:
+    attributes:
+
+      # Collector and Manifest version
+      - key: sw.k8s.agent.manifest.version
+        value: "1.0"
+        action: insert
+
+      # Cluster
+      - key: sw.k8s.cluster.uid
+        value: ${CLUSTER_UID}
+        action: insert
+
+      - key: k8s.cluster.name
+        value: ${CLUSTER_NAME}
+        action: insert
+
+      # Node
+      - key: k8s.node.name
+        value: ${NODE_NAME}
+        action: insert
+
+  batch:
+    send_batch_size: 8192
+    send_batch_max_size: 8192
+    timeout: 1s
+receivers:
+  filelog:
+    include: [ /var/log/pods/*/*/*.log ]
+    # Exclude collector container's logs. The file format is /var/log/pods/<namespace_name>_<pod_name>_<pod_uid>/<container_name>/<run_id>.log
+    exclude: [ "/var/log/pods/${POD_NAMESPACE}_${POD_NAME}*_*/swi-opentelemetry-collector/*.log" ]
+    start_at: beginning
+    include_file_path: true
+    include_file_name: false
+    operators:
+      # Find out which format is used by kubernetes
+      - type: router
+        id: get-format
+        routes:
+          - output: parser-docker
+            expr: 'body matches "^\\{"'
+          - output: parser-crio
+            expr: 'body matches "^[^ Z]+ "'
+          - output: parser-containerd
+            expr: 'body matches "^[^ Z]+Z"'
+      # Parse CRI-O format
+      - type: regex_parser
+        id: parser-crio
+        regex: '^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*)( |)(?P<log>.*)$'
+        output: merge-cri-lines
+        parse_to: body
+        timestamp:
+          parse_from: body.time
+          layout_type: gotime
+          layout: '2006-01-02T15:04:05.000000000-07:00'
+      # Parse CRI-Containerd format
+      - type: regex_parser
+        id: parser-containerd
+        regex: '^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*)( |)(?P<log>.*)$'
+        output: merge-cri-lines
+        parse_to: body
+        timestamp:
+          parse_from: body.time
+          layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+      # Parse Docker format
+      - type: json_parser
+        id: parser-docker
+        parse_to: body
+        output: merge-docker-lines
+        timestamp:
+          parse_from: body.time
+          layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+
+      # Merge log lines split by Docker logging driver.
+      - type: recombine
+        id: merge-docker-lines
+        source_identifier: attributes["log.file.path"]
+        output: merge-multiline-logs
+        combine_field: body.log
+        combine_with: ""
+        is_last_entry: body.log matches "\n$"
+
+      # Merge log lines split by CRI logging drivers.
+      - type: recombine
+        id: merge-cri-lines
+        source_identifier: attributes["log.file.path"]
+        output: merge-multiline-logs
+        combine_field: body.log
+        combine_with: ""
+        is_last_entry: body.logtag == "F"
+        overwrite_with: newest
+
+      # Merges incoming log records into multiline logs.
+      - type: recombine
+        id: merge-multiline-logs
+        output: extract-metadata-from-filepath
+        source_identifier: attributes["log.file.path"]
+        combine_field: body.log
+        combine_with: ""
+        is_first_entry: body.log matches "^\\[?\\d{4}-\\d{1,2}-\\d{1,2}.\\d{2}:\\d{2}:\\d{2}.*"
+
+      # Extract metadata from file path
+      - type: regex_parser
+        id: extract-metadata-from-filepath
+        regex: '^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]+)\/(?P<container_name>[^\._]+)\/(?P<run_id>\d+)\.log$'
+        parse_from: attributes["log.file.path"]
+
+      # Rename attributes
+      - type: move
+        id: move-attributes
+        from: body.stream
+        to: attributes["stream"]
+      - type: move
+        from: attributes.container_name
+        to: attributes["k8s.container.name"]
+      - type: move
+        from: attributes.namespace
+        to: attributes["k8s.namespace.name"]
+      - type: move
+        from: attributes.pod_name
+        to: attributes["k8s.pod.name"]
+      - type: move
+        from: attributes.run_id
+        to: attributes["run_id"]
+      - type: move
+        from: attributes.uid
+        to: attributes["k8s.pod.uid"]
+      - type: remove
+        field: attributes["log.file.path"]
+      - type: move
+        from: body.log
+        to: body
+      
+service:
+  extensions:
+    - health_check
+  pipelines:
+    logs:
+      exporters:
+        - otlp
+      processors:
+        - groupbyattrs/all
+        - resource
+        - batch
+      receivers:
+        - filelog
+  telemetry:
+    logs:
+      level: "info"
+    metrics:
+      address: 0.0.0.0:8888
diff --git a/build/nighthawk-swi-opentelemetry-collector.yaml b/build/nighthawk-swi-opentelemetry-collector.yaml
index 5b658092..942a0ca1 100644
--- a/build/nighthawk-swi-opentelemetry-collector.yaml
+++ b/build/nighthawk-swi-opentelemetry-collector.yaml
@@ -10,7 +10,8 @@ exporters:
 receivers:
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.51.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8seventsreceiver v0.51.0
-
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelogreceiver v0.51.0
+  
 processors:
   - import: go.opentelemetry.io/collector/processor/batchprocessor
     gomod: go.opentelemetry.io/collector v0.51.0
diff --git a/deploy/k8s/manifest.yaml b/deploy/k8s/manifest.yaml
index 8500704e..f5dc534e 100644
--- a/deploy/k8s/manifest.yaml
+++ b/deploy/k8s/manifest.yaml
@@ -145,4 +145,91 @@ spec:
           resources:
             limits:
               cpu: 256m
-              memory: 512Mi
\ No newline at end of file
+              memory: 512Mi
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: swi-opentelemetry-collector-logs
+spec:
+  selector:
+    matchLabels:
+      k8s-app: swi-opentelemetry-collector-logs
+  template:
+    metadata:
+      labels:
+        k8s-app: swi-opentelemetry-collector-logs
+    spec:
+      terminationGracePeriodSeconds: 30
+      securityContext:
+        ## In order to reliably read logs from mounted node logging paths, we need to run as root
+        fsGroup: 0
+        runAsUser: 0
+        runAsGroup: 0
+      containers:
+        - name: swi-opentelemetry-collector
+          image: "solarwinds/swi-opentelemetry-collector:latest"
+          imagePullPolicy: Always
+          command:
+            - /swi-otelcol
+            - --config=/opt/log-config.yaml
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: SOLARWINDS_API_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: solarwinds-api-token
+                  key: SOLARWINDS_API_TOKEN
+                  optional: true
+            - name: OTEL_ENVOY_ADDRESS
+              value: "<OTEL_ENVOY_ADDRESS>"
+            - name: CLUSTER_NAME
+              value: "<CLUSTER_NAME>"
+            - name: CLUSTER_UID
+              value: "<CLUSTER_UID>"
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 13133
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 13133
+          resources:
+            requests:
+              cpu: 100m
+              memory: 32Mi
+            limits:
+              cpu: 1000m
+              memory: 1Gi
+          volumeMounts:
+            - mountPath: /var/log/pods
+              name: varlogpods
+              readOnly: true
+            - mountPath: /var/log/containers
+              name: varlogcontainers
+              readOnly: true
+            - mountPath: /var/lib/docker/containers
+              name: varlibdockercontainers
+              readOnly: true
+      volumes:
+        - name: varlogpods
+          hostPath:
+            path: /var/log/pods
+        - name: varlogcontainers
+          hostPath:
+            path: /var/log/containers
+        - name: varlibdockercontainers
+          hostPath:
+            path: /var/lib/docker/containers
\ No newline at end of file
diff --git a/tests/deploy/base/kustomization.yaml b/tests/deploy/base/kustomization.yaml
index c898708d..781c536a 100644
--- a/tests/deploy/base/kustomization.yaml
+++ b/tests/deploy/base/kustomization.yaml
@@ -9,4 +9,5 @@ resources:
 
 patchesStrategicMerge:
   - patch-apply-env-variables.yaml
+  - patch-logs-apply-env-variables.yaml
   - patch-cluster-role-binding.yaml
diff --git a/tests/deploy/base/patch-logs-apply-env-variables.yaml b/tests/deploy/base/patch-logs-apply-env-variables.yaml
new file mode 100644
index 00000000..03dd3fc2
--- /dev/null
+++ b/tests/deploy/base/patch-logs-apply-env-variables.yaml
@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: swi-opentelemetry-collector-logs
+spec:
+  template:
+    spec:
+      containers:
+        - name: swi-opentelemetry-collector
+          image: nighthawk-swi-opentelemetry-collector
+          imagePullPolicy: Never
+          env:
+            - name: SOLARWINDS_API_TOKEN
+              $patch: delete
+            - name: OTEL_ENVOY_ADDRESS
+              value: "timeseries-mock-service:9082"
+            - name: OTEL_ENVOY_ADDRESS_TLS_INSECURE
+              value: "true"
+            - name: CLUSTER_NAME
+              value: "cluster name"
+            - name: CLUSTER_UID
+              value: "cluster-uid-123456789"

From 183cce0fc95cd45aa502052dd5d3518bb426f1d1 Mon Sep 17 00:00:00 2001
From: "tomas.panik" <tomas.panik@solarwinds.com>
Date: Fri, 1 Jul 2022 11:35:46 +0200
Subject: [PATCH 2/4] NH-9836: Adding separate time-series mock service for
 Logs

also adding basic test, verifying that some logs were recieved
---
 .github/workflows/buildAndDeploy.yml          |  18 +--
 tests/deploy/base/mocks.yaml                  | 106 ++++++++++++++++++
 .../base/patch-logs-apply-env-variables.yaml  |   2 +-
 3 files changed, 118 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/buildAndDeploy.yml b/.github/workflows/buildAndDeploy.yml
index 8a960595..c1175e51 100644
--- a/.github/workflows/buildAndDeploy.yml
+++ b/.github/workflows/buildAndDeploy.yml
@@ -42,17 +42,12 @@ jobs:
       - name: Deploy services using Skaffold
         run: skaffold run --skip-tests=true
 
-      - name: Tail Otel mock metrics for 60 seconds
+      - name: Tail Otel mock metrics,events,logs for 60 seconds
         run: |
           mkdir -p OtelLogs
           kubetail timeseries-mock-service --namespace monitoring > OtelLogs/metrics.log &
-          sleep 60
-
-      - name: Tail Otel mock events for 60 seconds
-        run: |
-          kubectl delete pods -l component=node-exporter -n monitoring
-          mkdir -p OtelLogs
           kubetail timeseries-mock-service --namespace monitoring > OtelLogs/events.log &
+          kubetail timeseries-logs-mock-service --namespace monitoring > OtelLogs/logs.log &
           sleep 60
 
       - name: Destroy environment
@@ -76,6 +71,15 @@ jobs:
             exit 1
           fi
 
+      - name: Evaluate logs collection functionality
+        run: |
+          if grep -q "#logs" "OtelLogs/logs.log"; then
+            echo "Some logs found"
+          else
+            echo "Logs are missing"
+            exit 1
+          fi
+
       - name: Capture Otel logs
         uses: actions/upload-artifact@v2
         if: failure() && hashFiles('OtelLogs') != '' # test existance of some report in `OtelLogs` folder
diff --git a/tests/deploy/base/mocks.yaml b/tests/deploy/base/mocks.yaml
index 7513954f..5e187f6c 100644
--- a/tests/deploy/base/mocks.yaml
+++ b/tests/deploy/base/mocks.yaml
@@ -40,6 +40,46 @@ data:
           - otlp
 ---
 apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: swi-opentelemetry-collector-endpoint-logs-mock
+  labels:
+    app.kubernetes.io/name: swi-opentelemetry-collector-endpoint-logs-mock
+    app.kubernetes.io/instance: swi-opentelemetry-collector-endpoint-logs-mock
+data:
+  relay: |
+    exporters:
+      logging:
+        loglevel: info
+    extensions:
+      health_check: {}
+      memory_ballast:
+        size_mib: "204"
+    processors:
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+            endpoint: 0.0.0.0:9082
+    service:
+      extensions:
+      - health_check
+      - memory_ballast
+      pipelines:
+        metrics:
+          exporters:
+          - logging
+          processors:
+          receivers:
+          - otlp
+        logs:
+          exporters:
+          - logging
+          processors:
+          receivers:
+          - otlp
+---
+apiVersion: v1
 kind: Service
 metadata:
   name: timeseries-mock-service
@@ -52,6 +92,19 @@ spec:
       targetPort: 9082
       name: grpc
 ---
+apiVersion: v1
+kind: Service
+metadata:
+  name: timeseries-logs-mock-service
+spec:
+  selector:
+    app: timeseries-logs-mock-service
+  ports:
+    - protocol: TCP
+      port: 9082
+      targetPort: 9082
+      name: grpc
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -104,5 +157,58 @@ spec:
             items:
               - key: relay
                 path: relay.yaml
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: timeseries-logs-mock-service
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: timeseries-logs-mock-service
+  template:
+    metadata:
+      labels:
+        app: timeseries-logs-mock-service
+    spec:
+      serviceAccountName: swi-opentelemetry-collector
+      securityContext:
+        {}
+      containers:
+        - name: opentelemetry-collector
+          command:
+            - /otelcol-contrib
+            - --config=/conf/relay.yaml
+          securityContext:
+            {}
+          image: "otel/opentelemetry-collector-contrib:0.43.0"
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: otlp
+              containerPort: 9082
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 13133
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 13133
+          resources:
+            limits:
+              cpu: 256m
+              memory: 512Mi
+          volumeMounts:
+            - mountPath: /conf
+              name: opentelemetry-collector-configmap
+      volumes:
+        - name: opentelemetry-collector-configmap
+          configMap:
+            name: swi-opentelemetry-collector-endpoint-logs-mock
+            items:
+              - key: relay
+                path: relay.yaml
 
 
diff --git a/tests/deploy/base/patch-logs-apply-env-variables.yaml b/tests/deploy/base/patch-logs-apply-env-variables.yaml
index 03dd3fc2..1d69b27d 100644
--- a/tests/deploy/base/patch-logs-apply-env-variables.yaml
+++ b/tests/deploy/base/patch-logs-apply-env-variables.yaml
@@ -13,7 +13,7 @@ spec:
             - name: SOLARWINDS_API_TOKEN
               $patch: delete
             - name: OTEL_ENVOY_ADDRESS
-              value: "timeseries-mock-service:9082"
+              value: "timeseries-logs-mock-service:9082"
             - name: OTEL_ENVOY_ADDRESS_TLS_INSECURE
               value: "true"
             - name: CLUSTER_NAME

From 62c8766bdc7644e37a8481f6bab97f6c76ec0295 Mon Sep 17 00:00:00 2001
From: "tomas.panik" <tomas.panik@solarwinds.com>
Date: Fri, 1 Jul 2022 11:47:31 +0200
Subject: [PATCH 3/4] NH-9834: Fixing CI pipeline

---
 .github/workflows/buildAndDeploy.yml | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/buildAndDeploy.yml b/.github/workflows/buildAndDeploy.yml
index c1175e51..64a20a99 100644
--- a/.github/workflows/buildAndDeploy.yml
+++ b/.github/workflows/buildAndDeploy.yml
@@ -42,16 +42,30 @@ jobs:
       - name: Deploy services using Skaffold
         run: skaffold run --skip-tests=true
 
-      - name: Tail Otel mock metrics,events,logs for 60 seconds
+      - name: Tail Otel mock metrics for 60 seconds
         run: |
           mkdir -p OtelLogs
           kubetail timeseries-mock-service --namespace monitoring > OtelLogs/metrics.log &
+          sleep 60
+
+      - name: Tail Otel mock events for 60 seconds
+        run: |
+          kubectl delete pods -l component=node-exporter -n monitoring
+          mkdir -p OtelLogs
           kubetail timeseries-mock-service --namespace monitoring > OtelLogs/events.log &
+          sleep 60
+
+      - name: Tail Otel mock logs for 60 seconds
+        run: |
+          mkdir -p OtelLogs
           kubetail timeseries-logs-mock-service --namespace monitoring > OtelLogs/logs.log &
           sleep 60
 
+      # Delete resources and wait some time for its termination
       - name: Destroy environment
-        run: skaffold delete
+        run: | 
+          skaffold delete
+          sleep 20
 
       - name: Evaluate metrics collection functionality
         run: |

From 590a0c02e0b658edf7fdf626b74eb1a300d12ea3 Mon Sep 17 00:00:00 2001
From: "tomas.panik" <tomas.panik@solarwinds.com>
Date: Fri, 1 Jul 2022 13:21:47 +0200
Subject: [PATCH 4/4] NH-9834: Adding filter to allow only logs from internal
 namespaces

To reduce amount of logs being sent by default if this feature goes to public without any further adjustments
---
 build/logs-otel-collector-config.yaml            | 11 +++++++++++
 build/nighthawk-swi-opentelemetry-collector.yaml |  1 +
 2 files changed, 12 insertions(+)

diff --git a/build/logs-otel-collector-config.yaml b/build/logs-otel-collector-config.yaml
index a9169556..13349697 100644
--- a/build/logs-otel-collector-config.yaml
+++ b/build/logs-otel-collector-config.yaml
@@ -9,6 +9,16 @@ extensions:
   health_check: {}
 
 processors:
+  # For more all the options about the filtering see https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/filterprocessor
+  filter:
+    logs:
+      include:
+          match_type: regexp
+          record_attributes:
+              # allow only system namespaces (kube-system, kube-public)
+              - key: k8s.namespace.name
+                value: ^kube-.*$
+
   groupbyattrs/all:
     keys:
       - k8s.container.name
@@ -157,6 +167,7 @@ service:
       exporters:
         - otlp
       processors:
+        - filter
         - groupbyattrs/all
         - resource
         - batch
diff --git a/build/nighthawk-swi-opentelemetry-collector.yaml b/build/nighthawk-swi-opentelemetry-collector.yaml
index 942a0ca1..e3c0722e 100644
--- a/build/nighthawk-swi-opentelemetry-collector.yaml
+++ b/build/nighthawk-swi-opentelemetry-collector.yaml
@@ -23,6 +23,7 @@ processors:
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatorateprocessor v0.51.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/cumulativetodeltaprocessor v0.51.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricsgenerationprocessor v0.51.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.51.0
   - gomod: github.com/solarwinds/nighthawk-im-k8s-monitor/processor/prometheustypeconverterprocessor v0.0.1
     path: "./src/processor/prometheustypeconverterprocessor"
   - gomod: github.com/solarwinds/nighthawk-im-k8s-monitor/processor/swmetricstransformprocessor v0.0.1