[NETOBSERV-58] Merge loki-exporter into kube-enricher (#10)

* merge goflow2-loki-exporter

* keep process reading

* loki process while enriching

* loadLokiConfig check fix

* mount loki-exporter-config volume

* removed main entry from gitignore

* removed unnecessary assert / require

* process loki before marshal

* extractTimestamp float64 fix

* skip json marshal on enrich

* getFloat64 logrus warning

.gitignore

@@ -1 +1,2 @@
 /kube-enricher
+coverage.out

.golangci.yml

@@ -0,0 +1,31 @@
+# TODO: agree with the team
+linters:
+  enable:
+    - deadcode
+    - errcheck
+    - errorlint
+    - cyclop
+    - errname
+    - exhaustive
+    - exportloopref
+    - gocritic
+    - goimports
+    - gosimple
+    - govet
+    - ineffassign
+    - revive
+    - structcheck
+    - stylecheck
+    - typecheck
+    - unused
+    - varcheck
+linters-settings:
+  stylecheck:
+    go: "1.17"
+  gocritic:
+    enabled-checks:
+      - hugeParam
+      - rangeExprCopy
+      - rangeValCopy
+      - indexAlloc
+      - deprecatedComment

Dockerfile

@@ -1,17 +1,11 @@
-FROM registry.access.redhat.com/ubi8/go-toolset:1.15.14-10 as builder
+FROM registry.access.redhat.com/ubi8/go-toolset:1.15.14-14 as builder
 ARG VERSION=""
 
 WORKDIR /opt/app-root/src
 
-COPY go.mod ./
-COPY go.sum ./
-
-RUN go mod download
-
 COPY . .
 
-
-RUN go build -ldflags "-X main.version=${VERSION}" -o kube-enricher cmd/kube-enricher/main.go
+RUN go build -ldflags "-X main.version=${VERSION}" -mod vendor -o kube-enricher cmd/kube-enricher/main.go
 
 FROM registry.access.redhat.com/ubi8/ubi-minimal:8.4-210
 

Makefile

@@ -1,5 +1,7 @@
 VERSION ?= dev
 IMAGE ?= quay.io/netobserv/goflow2-kube
+GOLANGCI_LINT_VERSION ?= v1.42.1
+COVERPROFILE ?= coverage.out
 
 ifeq (,$(shell which podman 2>/dev/null))
 OCI_BIN ?= docker
@@ -12,16 +14,31 @@ all: fmt build lint test
 fmt:
 	go fmt ./...
 
-lint:
-	golangci-lint run
+prereqs:
+	@echo "### Test if prerequisites are met, and installing missing dependencies"
+	test -f $(go env GOPATH)/bin/golangci-lint || go install github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION}
+	test -f $(go env GOPATH)/bin/staticcheck || go install honnef.co/go/tools/cmd/staticcheck@latest
+
+lint: prereqs
+	@echo "### Linting code"
+	# staticcheck does not work properly when invoked inside golangci-lint
+	staticcheck -f stylish ./...
+	golangci-lint run ./...
+
+verify: 
+	lint test
+
+.PHONY: 
+	prereqs lint image lint test verify
 
 test:
-	go test ./...
+	go test ./... -coverprofile ${COVERPROFILE}
 
 build:
 	go build -o kube-enricher cmd/kube-enricher/main.go
 
 image:
+	@echo "### Building container with ${OCI_BIN}"
 	$(OCI_BIN) build --build-arg VERSION="$(VERSION)" -t $(IMAGE):$(VERSION) .
 
 push:

README.md

@@ -93,6 +93,16 @@ make ovnk-deploy
 
 Finally check goflow's logs for output
 
+#### Legacy Netflow (v5)
+
+Similarly:
+
+```bash
+kubectl apply -f ./examples/goflow-kube-loki-nf5.yaml
+GF_IP=`kubectl get svc goflow-leg -ojsonpath='{.spec.clusterIP}'` && echo $GF_IP
+kubectl set env daemonset/ovnkube-node -c ovnkube-node -n ovn-kubernetes OVN_NETFLOW_TARGETS="$GF_IP:2056"
+```
+
 ### Run on OpenShift with OVNKubernetes network provider
 
 - Pre-requisite: make sure you have a running OpenShift cluster (4.8 at least) with `OVNKubernetes` set as the network provider.
@@ -109,3 +119,31 @@ or simply:
 ```bash
 make cno-deploy
 ```
+
+### Loki quickstart (helm)
+
+```bash
+helm upgrade --install loki grafana/loki-stack --set promtail.enabled=false
+helm install loki-grafana grafana/grafana
+kubectl get secret loki-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo
+kubectl port-forward svc/loki-grafana 3000:80
+```
+
+Open http://localhost:3000/
+Login with admin + printed password
+Add datasource => Loki =>
+http://loki:3100/
+
+Example of queries:
+
+- View raw logs:
+
+`{app="goflow2"}`
+
+- Top 10 sources by volumetry (1 min-rate):
+
+`topk(10, (sum by(SrcWorkload,SrcNamespace) ( rate({ app="goflow2" } | json | __error__="" | unwrap Bytes [1m]) )))`
+
+- Top 10 destinations for a given source (1 min-rate):
+
+`topk(10, (sum by(DstWorkload,DstNamespace) ( rate({ app="goflow2",SrcNamespace="default",SrcWorkload="goflow" } | json | __error__="" | unwrap Bytes [1m]) )))`

cmd/kube-enricher/main.go

@@ -18,6 +18,7 @@ import (
 	pbFormat "github.com/netobserv/goflow2-kube-enricher/pkg/format/pb"
 	"github.com/netobserv/goflow2-kube-enricher/pkg/reader"
 
+	"github.com/netobserv/goflow2-kube-enricher/pkg/export"
 	"github.com/sirupsen/logrus"
 	"k8s.io/client-go/rest"
 )
@@ -32,6 +33,7 @@ var (
 	listenAddress     = flag.String("listen", "", "listen address, if empty, will listen to stdin")
 	fieldsMapping     = flag.String("mapping", "SrcAddr=Src,DstAddr=Dst", "Mapping of fields containing IPs to prefixes for new fields")
 	kubeConfig        = flag.String("kubeconfig", "", "absolute path to the kubeconfig file")
+	lokiConfig        = flag.String("lokiconfig", "", "absolute path to the lokiconfig file")
 	logLevel          = flag.String("loglevel", "info", "Log level")
 	versionFlag       = flag.Bool("v", false, "Print version")
 	log               = logrus.WithField("module", app)
@@ -94,9 +96,15 @@ func main() {
 		log.Fatal(err)
 	}
 
+	log.Info("Creating loki...")
+	loki, err := export.NewLoki(loadLokiConfig())
+	if err != nil {
+		log.WithError(err).Fatal("Can't load Loki exporter")
+	}
+
 	r := reader.NewReader(in, log, mapping, clientset)
 	log.Info("Starting reader...")
-	r.Start()
+	r.Start(loki)
 }
 
 // loadKubeConfig fetches a given kubernetes configuration in the following order
@@ -107,11 +115,11 @@ func loadKubeConfig() *rest.Config {
 	var config *rest.Config
 	var err error
 	if kubeConfig != nil && *kubeConfig != "" {
-		log.Info("Using command line supplied kube config")
+		flog := log.WithField("configFile", *kubeConfig)
+		flog.Info("Using command line supplied kube config")
 		config, err = clientcmd.BuildConfigFromFlags("", *kubeConfig)
 		if err != nil {
-			log.WithError(err).WithField("kubeConfig", *kubeConfig).
-				Fatal("can't find provided kubeConfig param path")
+			flog.WithError(err).Fatal("Can't load kube config file")
 		}
 	} else if kfgPath := os.Getenv("KUBECONFIG"); kfgPath != "" {
 		log.Info("Using environment KUBECONFIG")
@@ -130,6 +138,34 @@ func loadKubeConfig() *rest.Config {
 	return config
 }
 
+// loadLokiConfig fetches a given kubernetes configuration in the following order
+// 1. path provided by the -lokiConfig CLI argument
+// 2. path provided by the LOKICONFIG environment variable
+// 3. default configuration
+func loadLokiConfig() *export.Config {
+	var config *export.Config
+	var err error
+	if lokiConfig != nil && *lokiConfig != "" {
+		flog := log.WithField("configFile", *lokiConfig)
+		flog.Info("Using command line supplied loki config")
+		config, err = export.LoadConfig(*lokiConfig)
+		if err != nil {
+			flog.WithError(err).Fatal("Can't load loki config file")
+		}
+	} else if lfgPath := os.Getenv("LOKICONFIG"); lfgPath != "" {
+		log.Info("Using environment LOKICONFIG")
+		config, err = export.LoadConfig(lfgPath)
+		if err != nil {
+			log.WithError(err).WithField("lokiConfig", lfgPath).
+				Fatal("can't find provided LOKICONFIG env path")
+		}
+	} else {
+		log.Info("Using loki default configuration")
+		config = export.DefaultConfig()
+	}
+	return config
+}
+
 func parseFieldMapping(in string) ([]reader.FieldMapping, error) {
 	var mapping []reader.FieldMapping
 	fields := strings.Split(in, ",")

examples/goflow-kube.yaml

@@ -15,32 +15,58 @@ spec:
       labels:
         app: goflow
     spec:
+      volumes:
+        - name: config-vol
+          configMap:
+            name: loki-exporter-config
       containers:
-      - command:
-        - /bin/sh
-        - -c
-        # - /goflow2 -loglevel "info" | /kube-enricher -loglevel "info" -sourceformat "json"
-        - /kube-enricher -loglevel "info" -stdinsourceformat "pb" -listen "netflow://:2055"
-        image: quay.io/netobserv/goflow2-kube:dev
-        imagePullPolicy: IfNotPresent
-        name: goflow
+        - command:
+            - /bin/sh
+            - -c
+            - /kube-enricher -listen netflow://:2055 -loglevel "trace" -lokiconfig /etc/loki-exporter/loki-exporter.yaml
+          image: quay.io/netobserv/goflow2-kube:dev
+          imagePullPolicy: IfNotPresent
+          name: goflow
+          resources:
+            limits:
+              memory: 512Mi
+              cpu: "1"
+          volumeMounts:
+              - mountPath: /etc/loki-exporter
+                name: config-vol
       serviceAccountName: goflow
 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: goflow
+  name: goflow 
   namespace: default
   labels:
     app: goflow
 spec:
   ports:
-  - port: 2055
-    protocol: UDP
+    - port: 2055
+      protocol: UDP
   selector:
     app: goflow
 ---
 apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: loki-exporter-config
+data:
+  loki-exporter.yaml: |
+    labels:
+      - SrcNamespace
+      - SrcWorkload
+      - DstNamespace
+      - DstWorkload
+    ignoreList:
+      - NextHop
+      - NextHopAS
+    printInput: true
+---
+apiVersion: v1
 kind: ServiceAccount
 metadata:
   labels:
@@ -59,9 +85,9 @@ roleRef:
   kind: ClusterRole
   name: goflow
 subjects:
-- kind: ServiceAccount
-  name: goflow
-  namespace: default
+  - kind: ServiceAccount
+    name: goflow
+    namespace: default
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -70,14 +96,14 @@ metadata:
     app: goflow
   name: goflow
 rules:
-- apiGroups:
-  - ""
-  - "apps"
-  resources:
-  - pods
-  - replicasets
-  - services
-  verbs:
-  - list
-  - get
-  - watch
+  - apiGroups:
+      - ""
+      - "apps"
+    resources:
+      - pods
+      - replicasets
+      - services
+    verbs:
+      - list
+      - get
+      - watch

go.mod

@@ -1,14 +1,19 @@
 module github.com/netobserv/goflow2-kube-enricher
 
-go 1.12
+go 1.15
 
 require (
-	github.com/mitchellh/mapstructure v1.1.2
+	github.com/go-kit/kit v0.12.0
+	github.com/json-iterator/go v1.1.12
+	github.com/mitchellh/mapstructure v1.4.2
+	github.com/netobserv/loki-client-go v0.0.0-20211018150932-cb17208397a9
 	github.com/netsampler/goflow2 v1.0.4
+	github.com/prometheus/common v0.31.1
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
-	google.golang.org/protobuf v1.25.0
-	k8s.io/api v0.20.1
-	k8s.io/apimachinery v0.20.1
-	k8s.io/client-go v0.20.1
+	google.golang.org/protobuf v1.27.1
+	gopkg.in/yaml.v2 v2.4.0
+	k8s.io/api v0.21.5
+	k8s.io/apimachinery v0.21.5
+	k8s.io/client-go v0.21.5
 )