Run check workloads based on asynq queues

stable/vulcan/Chart.yaml

@@ -24,19 +24,19 @@ appVersion: 1.0.0
 
 dependencies:
 - name: postgresql
-  version: 12.5.7
+  version: 16.2.2
   repository: oci://registry-1.docker.io/bitnamicharts
   condition: postgresql.enabled
 - name: redis
-  version: 17.11.5
+  version: 20.3.0
   repository: oci://registry-1.docker.io/bitnamicharts
   condition: redis.enabled
 - name: minio
-  version: 12.6.4
+  version: 14.8.5
   repository: oci://registry-1.docker.io/bitnamicharts
   condition: minio.enabled
 - name: localstack
-  version: 0.6.10
+  version: 0.6.17
   repository: https://localstack.github.io/helm-charts
   condition: localstack.enabled
 

stable/vulcan/templates/checks/_config.tpl

@@ -0,0 +1,17 @@
+{{- define "checks-config" -}}
+checktypes.json: |-
+  {{- $l := dict -}}
+  {{- range $key, $v := .Values.checks.checks -}}
+  {{- $_ := set $l $key (
+    dict 
+      "name" $key
+      "timeout" $v.timeout 
+      "options" (ternary nil (ternary $v.options (toJson $v.options) (kindIs "string" $v.options)) (empty $v.options))
+      "required_vars" $v.vars 
+      "image" (printf "%s:%s" $v.image.repository $v.image.tag)
+      "assets" $v.assets
+    )
+  -}}
+  {{- end -}}
+  {{ toPrettyJson $l | nindent 2 }}
+{{- end -}}

stable/vulcan/templates/checks/config.yaml

@@ -0,0 +1,2 @@
+{{- $_ := set .Values "comp" .Values.checks -}}
+{{- include "common-configmap" (merge (dict "Args" (dict "template" "checks-config")) . ) -}}

stable/vulcan/templates/checks/deployment.yaml

@@ -0,0 +1,240 @@
+{{- $_ := (set .Values "comp" .Values.checks) -}}
+{{- if .Values.checks.gateway.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "vulcan.fullname" $ }}-checkgw
+  labels: {{- include "vulcan.labels" $ | nindent 4 }}
+    app.kubernetes.io/name: {{ include "vulcan.fullname" $ }}-checkgw
+spec:
+  selector:
+    matchLabels: {{- include "vulcan.selectorLabels" $ | nindent 6 }}
+      app.kubernetes.io/name: {{ include "vulcan.fullname" $ }}-checkgw  
+  template:
+    metadata:
+      labels: {{- include "vulcan.podLabels" $ | nindent 8 }}
+        app.kubernetes.io/name: {{ include "vulcan.fullname" $ }}-checkgw
+      annotations:
+        checksum/config: {{ include "checks-config" . | sha256sum }}
+    spec:
+      {{- with .Values.checks.gateway.imagePullSecrets }}
+      imagePullSecrets:
+      {{- toYaml . | nindent 6 }}
+      {{- end }}
+      {{- with .Values.checks.gateway.serviceAccount }}
+      serviceAccountName: {{ . }}
+      {{- end }}
+      containers:
+      - name: checkgw
+        image:  {{ .Values.checks.gateway.image.repository }}:{{ .Values.checks.gateway.image.tag }}
+        imagePullPolicy: Always
+        env:
+          - name: REDIS_HOST
+            value: {{ include "vulcan.redis.host" $ | quote }}
+          - name: REDIS_USR
+            value: {{ include "vulcan.redis.username" $ | quote }}
+          - name: REDIS_PORT
+            value: {{ include "vulcan.redis.port" $ | quote }}
+          - name: REDIS_DB
+            value: {{ include "vulcan.redis.db" $ | quote }}
+          - name: AWS_DEFAULT_REGION
+            value: {{ $.Values.global.region | quote }}
+          - name: CHECKS_SQS_ARN
+            value: {{ tpl $.Values.checks.gateway.queueArn $ | quote }}
+          - name: AWS_SQS_ENDPOINT
+            value: {{ include "sqs.url" $ | quote }}
+          - name: PERSISTENCE_PORT
+            value: "8080"
+          - name: CHECKTYPES_PATH
+            value: /config/checktypes.json
+          {{- include "common-container-envs" $ | nindent 10 }}
+        volumeMounts:
+        - name: config-volume
+          mountPath: /config
+        ports:
+        - name: http
+          containerPort: 8080
+          protocol: TCP
+      volumes:
+        - name: config-volume
+          configMap:
+            name: {{ include "vulcan.fullname" $ }}-checks
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "vulcan.fullname" $ }}-checkgw
+  labels: {{- include "vulcan.labels" $ | nindent 4 }}
+    app.kubernetes.io/name: {{ include "vulcan.fullname" $ }}-checkgw
+spec:
+  ports:
+    - port: 80
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector: {{- include "vulcan.selectorLabels" $ | nindent 4 }}
+    app.kubernetes.io/name: {{ include "vulcan.fullname" $ }}-checkgw
+---
+{{- end }}
+{{- range $key, $value := .Values.checks.checks }}
+{{- if $value.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "vulcan.fullname" $ }}-check-{{ $key }}
+  labels: {{- include "vulcan.labels" $ | nindent 4 }}
+    app.kubernetes.io/name: {{ include "vulcan.fullname" $ }}-check-{{ $key }}
+spec:
+  selector:
+    matchLabels: {{- include "vulcan.selectorLabels" $ | nindent 6 }}
+      app.kubernetes.io/name: {{ include "vulcan.fullname" $ }}-check-{{ $key }}
+  template:
+    metadata:
+      labels: {{- include "vulcan.podLabels" $ | nindent 8 }}
+        app.kubernetes.io/name: {{ include "vulcan.fullname" $ }}-check-{{ $key }}
+      {{- with $.Values.checks.annotations }}
+      annotations:
+        {{ toYaml . | nindent 8 }}
+      {{- end }}
+    spec:
+      {{- with $value.imagePullSecrets }}
+      imagePullSecrets:
+      {{- toYaml . | nindent 6 }}
+      {{- end }}
+      {{- with $value.serviceAccount }}
+      serviceAccountName: {{ . }}
+      {{- end }}
+      containers:
+      - name: check
+        image:  {{ $value.image.repository }}:{{ $value.image.tag }}
+        imagePullPolicy: Always
+        env:
+        - name: VULCAN_HTTP_PORT
+          value: "8080"
+        {{- range $i, $v := $value.vars }}
+        - name: {{ $v }}
+          valueFrom:
+            secretKeyRef:
+              name: {{ $.Values.checks.secretVars }}
+              key: {{ $v }}
+        {{- end }}
+        ports:
+        - name: http
+          containerPort: 8080
+          protocol: TCP
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: http
+          initialDelaySeconds: 3
+          periodSeconds: 5
+          failureThreshold: 3
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: http
+          initialDelaySeconds: 3
+          periodSeconds: 5
+          failureThreshold: 5
+        {{- with $value.resources }}
+        resources:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+      initContainers:
+      - name: controller
+        image:  {{ $.Values.checks.controller.image.repository }}:{{ $.Values.checks.controller.image.tag }}
+        imagePullPolicy: Always
+        restartPolicy: Always
+        env:
+          - name: REDIS_HOST
+            value: {{ include "vulcan.redis.host" $ | quote }}
+          - name: REDIS_USR
+            value: {{ include "vulcan.redis.username" $ | quote }}
+          - name: REDIS_PORT
+            value: {{ include "vulcan.redis.port" $ | quote }}
+          - name: REDIS_DB
+            value: {{ include "vulcan.redis.db" $ | quote }}
+          - name: CHECK_NAME
+            value: {{ $key }}
+          - name: CHECK_ENDPOINT
+            value: http://localhost:8080
+          - name: CONCURRENCY
+            value: {{ $value.concurrency | quote }}
+          - name: AWS_DEFAULT_REGION
+            value: {{ $.Values.global.region | quote }}
+          - name: CHECKS_SQS_ARN
+            value: {{ tpl $.Values.checks.controller.queueArn $| quote }}
+          - name: AWS_S3_BUCKET
+            value: {{ $.Values.checks.controller.bucketReports | quote }}
+          - name: AWS_S3_PATH_TEMPLATE
+            value: {{ $.Values.checks.controller.pathTemplate | quote }}
+          - name: CHECKTYPE_NAME
+            value: {{ $value.image.repository }}
+          - name: CHECKTYPE_VERSION
+            value: {{ $value.image.tag }}
+          {{- include "common-container-envs" $ | nindent 10 }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "vulcan.fullname" $ }}-check-{{ $key }}
+  labels: {{- include "vulcan.labels" $ | nindent 4 }}
+    app.kubernetes.io/name: {{ include "vulcan.fullname" $ }}-check-{{ $key }}
+spec:
+  ports:
+    - port: 80
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector: {{- include "vulcan.selectorLabels" $ | nindent 4 }}
+    app.kubernetes.io/name: {{ include "vulcan.fullname" $ }}-check-{{ $key }}
+{{- if $.Values.checks.ingresEnabled }}
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ include "vulcan.fullname" $ }}-check-{{ $key }}
+  labels: {{- include "vulcan.labels" $ | nindent 4 }}
+    app.kubernetes.io/name: {{ include "vulcan.fullname" $ }}-check-{{ $key }}
+spec:
+  rules:
+  - host: {{ $key }}-check.localhost.direct
+    http:
+      paths:
+      - backend:
+          service:
+            name: {{ include "vulcan.fullname" $ }}-check-{{ $key }}
+            port:
+              number: 80
+        path: /
+        pathType: ImplementationSpecific
+  tls:
+  - hosts:
+    - {{ $key }}-checks.localhost.direct
+    secretName: localhost-direct-tls
+{{- end }}
+{{- if $.Values.checks.keda.enabled }}
+---
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: {{ include "vulcan.fullname" $ }}-check-{{ $key }}
+spec:
+  minReplicaCount: {{ $value.minReplicaCount | default 0 }}
+  maxReplicaCount: {{ $value.maxReplicaCount | default 5 }}
+  scaleTargetRef:
+    name: {{ include "vulcan.fullname" $ }}-check-{{ $key }}
+  triggers:
+    - type: redis
+      metadata:
+        address: {{ printf "%s:%s" (include "vulcan.redis.host" $) (include "vulcan.redis.port" $) }}
+        databaseIndex: {{ include "vulcan.redis.db" $ | quote }}
+        listName: {{ printf "asynq:{%s}:pending" $key }}
+        listLength: {{ $value.listLength | default 5 | quote }}
+        activationListLength: "1" # optional
+        enableTLS: "false" # optional
+        unsafeSsl: "false" # optional
+        usernameFromEnv: REDIS_USR
+{{- end }}
+{{- end }}
+{{- end }}

stable/vulcan/templates/scanengine/_config.tpl

@@ -1,3 +1,6 @@
 {{- define "scanengine-secrets" -}}
 PG_PASSWORD: {{ include "pg.encryptedPassword" . | quote }}
+{{- if .Values.comp.conf.queues.redisEnabled }}
+REDIS_PWD: {{ include "vulcan.redis.encryptedPassword" . | quote }}
+{{- end }}
 {{- end -}}

stable/vulcan/templates/scanengine/deployment.yaml

@@ -70,6 +70,16 @@ spec:
           - name: "QUEUES_{{ add1 $index }}_CHECKTYPES"
             value: {{ $value.checktypes | quote }}
           {{- end }}
+          {{- if .Values.comp.conf.queues.redisEnabled }}
+          - name: REDIS_HOST
+            value: {{ include "vulcan.redis.host" . | quote }}
+          - name: REDIS_USR
+            value: {{ include "vulcan.redis.username" . | quote }}
+          - name: REDIS_PORT
+            value: {{ include "vulcan.redis.port" . | quote }}
+          - name: REDIS_DB
+            value: {{ include "vulcan.redis.db" . | quote }}
+          {{- end }}
         {{- include "common-container-envs" . | nindent 10 }}
           envFrom:
           - secretRef:

stable/vulcan/values.yaml

@@ -483,6 +483,7 @@ scanengine:
     checksSNS:
       topicArn: arn:aws:sns:{{ .Values.global.region }}:{{ .Values.global.accountId }}:VulcanK8SChecks
     queues:
+      enableRedis: false
       default:
         arn: arn:aws:sqs:{{ .Values.global.region }}:{{ .Values.global.accountId }}:VulcanK8SV2ChecksGeneric
       # -- array of arn/checktypes
@@ -502,6 +503,13 @@ scanengine:
     <<: *db
     name: scanengine
 
+  redis:
+    host:
+    port:
+    username:
+    password:
+    db:
+
   dogstatsd: *dogstatsd
 
 
@@ -691,3 +699,62 @@ extraManifests: {}
   # config2: |
   #   apiVersion: v1
   #   ...
+
+checks:
+  enabled: false
+  name: checks
+
+  meta:
+    s3: true
+    sqs: true
+
+  redis:
+    host:
+    port:
+    username:
+    password:
+    db:
+
+  gateway:
+    enabled: false
+    imagePullSecrets: []
+    # - name: xxx
+    serviceAccount:
+    image:
+      repository: checks-gateway
+      tag: dev
+    queueArn:   # arn:aws:sqs:{{ .Values.global.region }}:{{ .Values.global.accountId }}:VulcanK8SV2ChecksGeneric
+  controller:
+    image:
+      repository: checks-controller
+      tag: dev
+    queueArn:  # arn:aws:sqs:{{ .Values.global.region }}:{{ .Values.global.accountId }}:VulcanK8SScanEngineCheckStatus
+    bucketReports: reports
+    pathTemplate: dt={{.Dt}}/{{.CheckID}}/{{.CheckID}}.json
+  annotations:
+  keda:
+    enabled: false
+  ingressEnabled: false
+
+  # secretVars is a secret with all the vars for the checks.
+  # TODO: Define at check level to allow more flexibility, or just a raw extraEnv.
+  secretVars: vulcan-agent-vars
+
+  checks:
+    vulca-sleep:
+      enabled: false
+      image:
+        repository: vulcansec/vulcan-sleep
+        tag: checkshttp
+      serviceAccount: ""
+      imagePullSecrets:
+      - name: foo
+      concurrency: 3
+      vars: []
+      assets: []
+      timeout: 20
+      options:   # OR options: "{\"sleep_time\":10}"
+        sleep_time: 10
+      minReplicaCount: 0
+      maxReplicaCount: 5
+      resources: {}