executors: deploy on k8s (#4224)

* deploy executor on k8s
sourcegraph · Jan 30, 2023 · 03c0904 · 03c0904
1 parent 18373c0
commit 03c0904
Show file tree

Hide file tree

Showing 7 changed files with 257 additions and 0 deletions.
diff --git a/configure/executors/README.md b/configure/executors/README.md
@@ -0,0 +1,26 @@
+# Executors
+
+Executors are Sourcegraph’s solution for running untrusted code in a secure and controllable way. For more information on executors and how they are used see the Executors [documentation](https://docs.sourcegraph.com/admin/executors)
+
+## Deploying
+
+This directory contains manifests for the optional deployment of Sourcegraph Executors on Kubernetes.
+
+It is expected that all components contained in this directory and any subdirectories are deployed to ensure full functionality and best performance.
+
+The following components will deployed:
+
+- [Executor Deployment](./executor/executor.Deployment.yaml) An Executor replica with a Docker sidecar to run isolated batch changes and auto-indexing jobs. This deployment requires a [privileged security context](https://kubernetes.io/docs/concepts/security/pod-security-standards/).
+- [Executor Service](./executor/executor.Service.yaml) A headless service for executor metrics access. Executors are not externally accessible.
+- [Docker ConfigMap](./executor/docker-daemon.ConfigMap.yaml) configuration for the docker sidecar to use the pull-through cache.
+- [Private docker registory]
+  - [Registry Deployment](./private-docker-registry/private-docker-registry.Deployment.yaml) A private docker registry configured as a pull-through cache to avoid docker hub rate limiting.
+  - [Registry Service](./private-docker-registry/private-docker-registry.Service.yaml) A service to access the private-docker-registry.
+  - [Registry Persistent Volume](./private-docker-registry/private-docker-registry.PersistentVolumeClaim.yaml) A volume to store images in the private-docker-registry.
+
+To apply these manifests, run the following command:
+
+```bash
+kubectl apply -f . --recursive
+```
+
diff --git a/configure/executors/executor/docker-daemon.ConfigMap.yaml b/configure/executors/executor/docker-daemon.ConfigMap.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+data:
+  daemon.json: |
+    { "insecure-registries":["private-docker-registry:5000"] }
+
+kind: ConfigMap
+metadata:
+  labels:
+    app: executor
+    deploy: sourcegraph
+    sourcegraph-resource-requires: no-cluster-admin
+    app.kubernetes.io/component: executor    
+  name: docker-config
diff --git a/configure/executors/executor/executor.Deployment.yaml b/configure/executors/executor/executor.Deployment.yaml
@@ -0,0 +1,116 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: executor
+  annotations:
+    description: Runs sourcegraph executor replicas for batch chanes and codeintel auto indexing.
+    kubectl.kubernetes.io/default-container: executor
+  labels:
+    deploy: sourcegraph
+    sourcegraph-resource-requires: no-cluster-admin
+    app.kubernetes.io/component: executor
+spec:
+  selector:
+    matchLabels:
+      app: executor
+  minReadySeconds: 10
+  replicas: 1
+  revisionHistoryLimit: 10
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: executor
+    spec:
+      containers:
+        - name: executor
+          image: index.docker.io/sourcegraph/executor:insiders@sha256:dfeef2e31d6c7b9bc3e5bf581180668f7c033ffcf1fff9d3d6380b7b998d4c2b
+          imagePullPolicy: Always
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: debug
+              scheme: HTTP
+            initialDelaySeconds: 60
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: debug
+              scheme: HTTP
+            periodSeconds: 5
+            timeoutSeconds: 5
+          ports:
+            - containerPort: 6060
+              name: debug
+          terminationMessagePolicy: FallbackToLogsOnError
+          # Refer to https://docs.sourcegraph.com/admin/deploy_executors_binary#step-2-setup-environment-variables on how to populate these variables
+          env:
+            - name: EXECUTOR_FRONTEND_URL
+              value:
+            - name: EXECUTOR_FRONTEND_PASSWORD
+              value:
+            - name: EXECUTOR_USE_FIRECRACKER
+              value: "false"
+            - name: EXECUTOR_QUEUE_NAME
+              value:
+            - name: EXECUTOR_JOB_NUM_CPUS
+              value: "0"
+            - name: EXECUTOR_JOB_MEMORY
+              value: "0"
+            - name: DOCKER_HOST
+              value: tcp://localhost:2375
+            # Note: Must match the mount point shared with the dind sidecar
+            - name: TMPDIR
+              value: /scratch
+          volumeMounts:
+            - mountPath: /scratch
+              name: executor-scratch
+        - name: dind
+          image: docker:20.10.22-dind@sha256:03f2d563100b9776283de1e18f10a1f0b66d2fdc7918831bf8db1cda767d6b37
+          securityContext:
+            privileged: true
+          command:
+            - 'dockerd'
+            - '--tls=false'
+            - '--mtu=1200'
+            - '--registry-mirror=http://private-docker-registry:5000'
+            - '--host=tcp://0.0.0.0:2375'
+          livenessProbe:
+            tcpSocket:
+                  port: 2375
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            failureThreshold: 5
+          readinessProbe:
+            tcpSocket:
+              port: 2375
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            failureThreshold: 5
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.name
+          ports:
+            - containerPort: 2375
+              protocol: TCP
+          volumeMounts:
+            - mountPath: /scratch
+              name: executor-scratch
+            - mountPath: /etc/docker/daemon.json
+              subPath: daemon.json
+              name: docker-config
+      volumes:
+        - name: executor-scratch
+          emptyDir: {}
+        - name: docker-config
+          configMap:
+            defaultMode: 420
+            name: docker-config
diff --git a/configure/executors/executor/executor.Service.yaml b/configure/executors/executor/executor.Service.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    prometheus.io/port: "6060"
+    sourcegraph.prometheus/scrape: "true"
+  labels:
+    app: executor
+    deploy: sourcegraph
+    sourcegraph-resource-requires: no-cluster-admin
+    app.kubernetes.io/component: executor
+  name: executor
+spec:
+  ports:
+  - name: debug
+    port: 6060
+    targetPort: debug
+  selector:
+    app: executor
+  type: ClusterIP
diff --git a/configure/executors/private-docker-registry/private-docker-registry.Deployment.yaml b/configure/executors/private-docker-registry/private-docker-registry.Deployment.yaml
@@ -0,0 +1,48 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: private-docker-registry
+  labels:
+    deploy: sourcegraph
+    sourcegraph-resource-requires: no-cluster-admin
+    app.kubernetes.io/component: private-docker-registry
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: private-docker-registry
+  template:
+    spec:
+      containers:
+        - image: index.docker.io/registry:2
+          name: private-docker-registry
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: REGISTRY_PROXY_REMOTEURL
+              value: http://registry-1.docker.io
+          ports:
+            - containerPort: 5000
+              name: registry
+          livenessProbe:
+            httpGet:
+              path: /
+              port: registry
+              scheme: HTTP
+            initialDelaySeconds: 5
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /
+              port: registry
+              scheme: HTTP
+            periodSeconds: 5
+            timeoutSeconds: 5         
+          volumeMounts:
+            - mountPath: /var/lib/registry
+              name: cache
+      volumes:
+        - name: cache
+          persistentVolumeClaim:
+            claimName: private-docker-registry
diff --git a/...gure/executors/private-docker-registry/private-docker-registry.PersistentVolumeClaim.yaml b/...gure/executors/private-docker-registry/private-docker-registry.PersistentVolumeClaim.yaml
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: private-docker-registry
+  labels:
+    deploy: sourcegraph
+    sourcegraph-resource-requires: no-cluster-admin
+    app.kubernetes.io/component: private-docker-registry  
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      # To be adjusted based on the number and size of images used in batch changes and auto-indexing
+      storage: 100Gi
+  storageClassName: sourcegraph
diff --git a/configure/executors/private-docker-registry/private-docker-registry.Service.yaml b/configure/executors/private-docker-registry/private-docker-registry.Service.yaml
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    deploy: sourcegraph
+    sourcegraph-resource-requires: no-cluster-admin
+    app.kubernetes.io/component: private-docker-registry
+  name: private-docker-registry
+  namespace: default
+spec:
+  ports:
+    - name: http
+      port: 5000
+      protocol: TCP
+      targetPort: 5000
+  selector:
+    app: private-docker-registry
+  type: ClusterIP