Deploy with yaml (#58)

.buildkite/install-yj.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+VERSION=1.0.0
+
+if [ ! $(which yj) ]; then
+    echo "Installing yj to /usr/local/bin"
+    wget https://github.com/sourcegraph/yj/releases/download/v${VERSION}/yj-${VERSION}-linux-amd64 -O /usr/local/bin/yj
+    chmod a+x /usr/local/bin/yj
+fi

.buildkite/pipeline.yml

@@ -1,5 +1,5 @@
 steps:
-  - label: ":helm:"
-    command: .buildkite/verify-generated.sh
   - label: ":k8s:"
-    command: .buildkite/verify-valid.sh
+    command: .buildkite/verify-yaml.sh
+  - label: ":k8s:"
+    command: .buildkite/verify-label.sh

.buildkite/verify-label.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -ex
+
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+
+.buildkite/install-yj.sh
+
+if find base -name "*.yaml" -exec sh -c "cat {} | yj | jq --raw-output '.metadata.labels.deploy'" \; | tee /tmp/deploy-label | grep -v sourcegraph; then
+    echo "> There exists a yaml file in base/ that does not contain .metadata.labels.deploy == sourcegraph"
+    echo "> Run the following command to fix:"
+    echo "find base/ -name \"*.yaml\" -exec sh -c \"cat {} | yj | jq '.metadata.labels.deploy = \\\"sourcegraph\\\"' | jy -o {}\" \;"
+
+    exit 1
+fi

.buildkite/verify-yaml.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -ex
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+
+.buildkite/install-kubeval.sh
+
+find base -name '*.yaml' -exec kubeval {} +
+find configure -name '*.yaml' -exec kubeval {} +
+
+.buildkite/verify-label.sh

.gitignore

@@ -0,0 +1,2 @@
+generated/
+

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+  "[yaml]": {
+    "editor.formatOnSave": false
+  }
+}

.vscode/tags

@@ -0,0 +1,7 @@
+!_TAG_FILE_FORMAT	2	/extended format; --format=1 will not append ;" to lines/
+!_TAG_FILE_SORTED	1	/0=unsorted, 1=sorted, 2=foldcase/
+!_TAG_OUTPUT_MODE	u-ctags	/u-ctags or e-ctags/
+!_TAG_PROGRAM_AUTHOR	Universal Ctags Team	//
+!_TAG_PROGRAM_NAME	Universal Ctags	/Derived from Exuberant Ctags/
+!_TAG_PROGRAM_URL	https://ctags.io/	/official site/
+!_TAG_PROGRAM_VERSION	0.0.0	/e5c86f7/

README.dev.md

@@ -2,24 +2,28 @@
 
 ## Cutting a release
 
-- Make the desired changes to this repository. Most commonly, this involves updating the Docker image versions in `values.yaml` to match the tagged version you are releasing. For images like language servers, you should look at our
-  [container registry](https://console.cloud.google.com/gcr/images/sourcegraph-dev?project=sourcegraph-dev) to see what the latest versions are.
+- Make the desired changes to this repository. Most commonly, this involves updating the Docker image versions in `*.Deployment.yaml` to match the tagged version you are releasing. You should look at our
+  [DockerHub repositories](https://hub.docker.com/r/sourcegraph/) to see what the latest versions are.
 
-- Run `./generate.sh`.
 - Open a PR and wait for buildkite to pass and for your changes to be approved, then merge and check out master.
-- Test this release on a fresh cluster before cutting a real release. Cut a test release (`LATEST=false ./release $TEST_VERSION`), then use the instructions [in this installation doc](docs/install.md) to set up a new cluster. (Make sure that you use the helm charts located at https://github.com/sourcegraph/datacenter/archive/$TEST_VERSION.tar.gz).
-- Run `./release.sh $VERSION`.
-  - If this is a patch version to a previous major/minor version, use `LATEST=false ./release.sh $VERSION`.
-  - If this is a release candidate, run `LATEST=false ./release {$VERSION}-pre${N}` where `N` starts
-    at 0 and increments as you test/cut new versions.
-- [Update the `latestReleaseDataCenterBuild` value in `sourcegraph/sourcegraph`](https://sourcegraph.sgdev.org/github.com/sourcegraph/sourcegraph/-/blob/cmd/server/README.md#5-notify-existing-instances-that-an-update-is-available)
+- Test what is currently checked in to master by [installing](docs/install.md) Sourcegraph on fresh cluster.
+- Create a git tag and push it to the repository:
+
+  ```bash
+  VERSION = vX.Y.Z
 
-See `./release.sh --help` for information on what the script does.
+  # If this is a release candidate: VERSION = `vX.Y.Z-pre${N}` (where `N` starts at 0 and increments as you test/cut new versions)
 
+  # 🚨 Make sure that you have the commit that you want to tag as $VERSION checked out!
 
+  git tag $VERSION
+  git push origin $VERSION
+  ```
 
-## Development tips
+- Cut the legacy Helm version of the release (this step will be deprecated after the next iteration):
+  - checkout [deploy-sourcegraph@helm-legacy](https://github.com/sourcegraph/deploy-sourcegraph/tree/helm-legacy)
+  - update the image tags in [yalues.yaml](https://github.com/sourcegraph/deploy-sourcegraph/blob/helm-legacy/values.yaml)
+  - run [generate.sh](https://github.com/sourcegraph/deploy-sourcegraph/blob/helm-legacy/generate.sh)
+  - run [release.sh](https://github.com/sourcegraph/deploy-sourcegraph/blob/helm-legacy/release.sh)
 
-- Whitespace in Helm templates can be tricky. When embedding the contents of a helper template, use
-  the form `{{- include "myHelper" $arg | nindent $indent }}`.
-  - If the output of the helper template could be empty, use `{{- include "myHelper" $arg | nindent $indent | trimSuffix "\n " }}`, where the argument to `trimSuffix` has $indent spaces.
+- [Update the `latestReleaseDataCenterBuild` value in `sourcegraph/sourcegraph`](https://sourcegraph.sgdev.org/github.com/sourcegraph/sourcegraph/-/blob/cmd/server/README.md#5-notify-existing-instances-that-an-update-is-available)

README.md

@@ -1,25 +1,26 @@
+> 🚨 If you are updating from a 2.10.x or previous deployment, follow the migration steps in [docs/migrate.md](docs/migrate.md).
+
 # Sourcegraph Data Center
 
 Sourcegraph Data Center is for organizations that need highly scalable and available code search and
 code intelligence. This repository contains documentation for creating, updating, and maintaining a
-Sourcegraph Data Center cluster.
+Sourcegraph Data Center cluster using Kubernetes.
 
 For product and [pricing](https://about.sourcegraph.com/pricing/) information,
 visit [about.sourcegraph.com](https://about.sourcegraph.com)
 or [contact us](https://about.sourcegraph.com/contact/sales) for more information. If you're just
 starting out, we recommend installing [Sourcegraph](https://about.sourcegraph.com/docs) on a single
 node first. Migrating to Data Center is easy when you're ready.
 
-- [Installation](docs/install.md)
+- [Installing](docs/install.md)
+- [Configuring](docs/configure.md)
 - [Updating](docs/update.md)
-- [Scaling](/docs/scale.md)
-- [Troubleshooting](/docs/troubleshoot.md)
-- [Admin guide](/docs/admin-guide.md) - useful guide for Sourcegraph admins
-  - [Prometheus metrics](/docs/prom-metrics.md) - list of all Prometheus metrics that can be used for
+- [Scaling](docs/scale.md)
+- [Troubleshooting](docs/troubleshoot.md)
+- [Admin guide](docs/admin-guide.md) - useful guide for Sourcegraph admins
+  - [Prometheus metrics](docs/prom-metrics.md) - list of all Prometheus metrics that can be used for
     application performance monitoring
 
 ## Contributing
 
-We've made this Helm chart open source to better serve our customers' needs. If there is a
-configuration point or Kubernetes option you would like to add, we would love to incorporate
-it. Pull requests and issues are reviewed and responded to quickly.
+We've made our deployment configurations open source to better serve our customers' needs. If there is anything we can do to make deploying Sourcegraph easier just open an issue or a pull request and we will respond promptly!

examples/aws/generated/sourcegraph/templates/backend.Service.yaml → base/backend.Service.yaml

@@ -1,11 +1,12 @@
-##---
-# Source: sourcegraph/templates/backend.Service.yaml
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    description: Dummy service that prevents backend pods from being scheduled on
+      the same node if possible.
   labels:
-    group: backend
     deploy: sourcegraph
+    group: backend
   name: backend
 spec:
   clusterIP: None

base/config-file.ConfigMap.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+data:
+  config.json: |-
+    {
+      // Publicly accessible URL to web app (e.g., what you type into your browser).
+      "appURL": "http://localhost:3080",
+
+      // The authentication provider to use for identifying and signing in users. Only one entry is supported.
+      //
+      // The builtin auth provider with signup disallowed (shown below) means that after the initial site admin signs in, all other users must be invited.
+      //
+      // Other providers are documented here:
+      // https://about.sourcegraph.com/docs/config/site#authproviders-array
+      "auth.providers": [{"type": "builtin", "allowSignup": false}],
+    }
+kind: ConfigMap
+metadata:
+  labels:
+    deploy: sourcegraph
+  name: config-file

examples/aws/generated/sourcegraph/templates/frontend/sourcegraph-frontend.Deployment.yaml → base/frontend/sourcegraph-frontend.Deployment.yaml

@@ -1,5 +1,3 @@
-##---
-# Source: sourcegraph/templates/frontend/sourcegraph-frontend.Deployment.yaml
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
@@ -26,8 +24,8 @@ spec:
       - args:
         - serve
         env:
-        - name: CONFIG_FILE_HASH
-          value: c382e72e1a7278843803db4ea55ff0bdb2dafe8200247c120189c04ceb01d280
+        - name: SOURCEGRAPH_CONFIG_FILE
+          value: /etc/sourcegraph/config.json
         - name: DEPLOY_TYPE
           value: datacenter
         - name: LSP_PROXY
@@ -44,19 +42,27 @@ spec:
           value: sg
         - name: PUBLIC_REPO_REDIRECTS
           value: "true"
-        - name: SOURCEGRAPH_CONFIG_FILE
-          value: /etc/sourcegraph/config.json
-        - name: SRC_APP_URL
-          value: http://localhost:3080
         - name: SRC_GIT_SERVERS
-          value: gitserver-1:3178
-        image: sourcegraph/frontend:2.10.1
+          value: gitserver-0.gitserver:3178
+        # See the customization guide (../../../docs/configure.md) for information 
+        # about how to configure Sourcegraph to use TLS
+        # - name: TLS_CERT
+        #   valueFrom:
+        #     secretKeyRef:
+        #       key: cert
+        #       name: tls
+        # - name: TLS_KEY
+        #   valueFrom:
+        #     secretKeyRef:
+        #       key: key
+        #       name: tls
+        image: sourcegraph/frontend:2.11.0
         livenessProbe:
           httpGet:
             path: /healthz
             port: http
             scheme: HTTP
-          initialDelaySeconds: 5
+          initialDelaySeconds: 60
           timeoutSeconds: 5
         name: frontend
         ports:
@@ -69,10 +75,10 @@ spec:
         resources:
           limits:
             cpu: "2"
-            memory: "2G"
+            memory: 2G
           requests:
             cpu: "2"
-            memory: "1G"
+            memory: 1G
         volumeMounts:
         - mountPath: /etc/sourcegraph
           name: sg-config

templates/frontend/sourcegraph-frontend.Role.yaml → base/frontend/sourcegraph-frontend.Role.yaml

@@ -1,10 +1,9 @@
-{{- if ne .Values.cluster.rbac "disabled" -}}
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: Role
 metadata:
-  name: sourcegraph-frontend
   labels:
     deploy: sourcegraph
+  name: sourcegraph-frontend
 rules:
 - apiGroups:
   - ""
@@ -14,4 +13,3 @@ rules:
   - get
   - list
   - watch
-{{ end -}}

templates/frontend/sourcegraph-frontend.RoleBinding.yaml → base/frontend/sourcegraph-frontend.RoleBinding.yaml

@@ -1,15 +1,13 @@
-{{- if ne .Values.cluster.rbac "disabled" }}
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: RoleBinding
 metadata:
-  name: sourcegraph-frontend
-  labels:   
+  labels:
     deploy: sourcegraph
+  name: sourcegraph-frontend
 roleRef:
   apiGroup: ""
   kind: Role
   name: sourcegraph-frontend
 subjects:
 - kind: ServiceAccount
   name: sourcegraph-frontend
-{{ end -}}

examples/custom-env-vars/generated/sourcegraph/templates/frontend/sourcegraph-frontend.Service.yaml → base/frontend/sourcegraph-frontend.Service.yaml

@@ -1,5 +1,3 @@
-##---
-# Source: sourcegraph/templates/frontend/sourcegraph-frontend.Service.yaml
 apiVersion: v1
 kind: Service
 metadata:
@@ -16,6 +14,10 @@ spec:
     nodePort: 30080
     port: 30080
     targetPort: http
+  - name: https
+    nodePort: 30081
+    port: 30081
+    targetPort: https
   selector:
     app: sourcegraph-frontend
   type: NodePort

templates/frontend/sourcegraph-frontend.ServiceAccount.yaml → base/frontend/sourcegraph-frontend.ServiceAccount.yaml

@@ -3,6 +3,6 @@ imagePullSecrets:
 - name: docker-registry
 kind: ServiceAccount
 metadata:
-  name: sourcegraph-frontend
-  labels:   
+  labels:
     deploy: sourcegraph
+  name: sourcegraph-frontend