diff --git a/.gitignore b/.gitignore index 48403d76..1f4370b5 100644 --- a/.gitignore +++ b/.gitignore @@ -268,3 +268,8 @@ __pycache__/ # all-contributors-cli node_modules + +### Helm +helm/*/myvalues.yaml +# Dependencies +helm/*/charts \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index c07dbee4..8ccfa5ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,7 @@ WORKDIR /nem-per-feina/ RUN apt-get update \ && apt-get install --no-install-recommends -qy wait-for-it \ + build-essential \ && rm -rf /var/lib/apt/list/* COPY requirements.txt ./ diff --git a/Dockerfile.test b/Dockerfile.test index ab23f0e2..9bfd58c2 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -5,6 +5,7 @@ WORKDIR /nem-per-feina/ RUN apt-get update \ && apt-get install --no-install-recommends -qy wait-for-it \ + build-essential \ && rm -rf /var/lib/apt/list/* COPY requirements.txt ./ diff --git a/README.md b/README.md index 09696a16..b86d195a 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Show your support by 🌟 the project!! - [The project](#the-project) +- [Deployment](#deployment) - [Contributing](#contributing) - [Tech stack](#tech-stack) - [Setup backend development environment](#setup-backend-development-environment) @@ -40,7 +41,13 @@ We forked the open source project [django-job-portal](https://github.com/manjuru ### Kubernetes deployment -To deploy to a k8s cluster see the [k8s manifests](/manifests/README.md) +#### Using k8s manifests + +To deploy to a k8s cluster using core manifests see the [NPF k8s manifests](/manifests/README.md) + +#### Using Helm Chart + +To deploy to a k8s cluster see the [NPF Helm Chart](/helm/npf/README.md) diff --git a/helm/apf/.helmignore b/helm/apf/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/helm/apf/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/apf/Chart.lock b/helm/apf/Chart.lock new file mode 100644 index 00000000..571f3289 --- /dev/null +++ b/helm/apf/Chart.lock @@ -0,0 +1,9 @@ +dependencies: +- name: redis + repository: https://charts.bitnami.com/bitnami + version: 11.0.6 +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 9.8.1 +digest: sha256:7ffea47877067e199faa37e7cd087d175e79657220cc27ad70b560448a1ea34d +generated: "2020-10-30T13:02:10.028763072+01:00" diff --git a/helm/apf/Chart.yaml b/helm/apf/Chart.yaml new file mode 100644 index 00000000..97e4eead --- /dev/null +++ b/helm/apf/Chart.yaml @@ -0,0 +1,22 @@ +apiVersion: v2 +name: npf +description: Nem per feina Chart +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.2.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: 1.16.0 + +dependencies: + - name: redis + version: 11.0.6 + repository: https://charts.bitnami.com/bitnami + - name: postgresql + version: 9.8.1 + repository: https://charts.bitnami.com/bitnami diff --git a/helm/apf/README.md b/helm/apf/README.md new file mode 100644 index 00000000..8e2d6cc3 --- /dev/null +++ b/helm/apf/README.md @@ -0,0 +1,65 @@ +# Helm Chart + +This provides our NPF Helm Chart + +## Deployment + +Create your deployment values file overriding [base values](values.yaml): + +```bash +cat values-prod.yaml + +baseUrl: https://nemperfeina.cat +baseEmail: it@geekscat.org + +app: + image: geekscat/nem-per-feina + tag: latest + workers: 4 + threads: 2 + debug: "False" + log_level: INFO + secretKey: this-is-not-an-strong-password + replicaCount: 1 + +postgresql: + postgresqlPassword: this-is-not-an-strong-password + +celery: + stdouts_level: INFO + replicaCount: 1 + +ingress: + hosts: + - host: nemperfeina.cat + paths: + - path: / + servicePort: 8000 + +... +``` + +```bash +# Create namespace +kubectl create ns $SOME_NAME + +# Install Chart with our custom overrides +helm -n $SOME_NAME install $SOME_NAME helm/apf -f values-prod.yaml +``` + +## Troubleshooting + +Ingress API changed before 1.18+, so remember to define your IngressClass (and opt mark it as defautl) with something like: + +```yaml +apiVersion: networking.k8s.io/v1beta1 +kind: IngressClass +metadata: + name: traefik + annotations: + ingressclass.kubernetes.io/is-default-class: 'true' +spec: + controller: traefik.io/ingress-controller +``` + +If you don't want to define it as `default-class`, pass `ingressClassName: traefik` to your `Ingress.spec` diff --git a/helm/apf/templates/_helpers.tpl b/helm/apf/templates/_helpers.tpl new file mode 100644 index 00000000..b341ec5b --- /dev/null +++ b/helm/apf/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "apf.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "apf.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "apf.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "apf.labels" -}} +helm.sh/chart: {{ include "apf.chart" . }} +{{ include "apf.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "apf.selectorLabels" -}} +app.kubernetes.io/name: {{ include "apf.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "apf.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "apf.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/helm/apf/templates/certificate.yaml b/helm/apf/templates/certificate.yaml new file mode 100644 index 00000000..6204fbeb --- /dev/null +++ b/helm/apf/templates/certificate.yaml @@ -0,0 +1,12 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "apf.fullname" . }}-cert +spec: + secretName: {{ include "apf.fullname" . }}-cert + issuerRef: + kind: ClusterIssuer + name: {{ .Values.certificate.issuername }} + commonName: {{ .Values.certificate.hostname }} + dnsNames: + - {{ .Values.certificate.hostname }} \ No newline at end of file diff --git a/helm/apf/templates/deployments/app.yaml b/helm/apf/templates/deployments/app.yaml new file mode 100644 index 00000000..dd5f0fa9 --- /dev/null +++ b/helm/apf/templates/deployments/app.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "apf.fullname" . }} + labels: + {{- include "apf.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.app.replicaCount }} + selector: + matchLabels: + {{- include "apf.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "apf.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "apf.serviceAccountName" . }} + containers: + - name: {{ .Chart.Name }}-celery + image: "{{ .Values.app.image }}:{{ .Values.app.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.app.pullPolicy }} + command: + - invoke + - migrate + - runserver + # command: + # - invoke + # - migrate + # - uwsgi + # - --port + # - {{ .Values.app.container.port | quote }} + # - -w + # - {{ .Values.app.workers | quote }} + # - -t + # - {{ .Values.app.threads | quote }} + envFrom: + - secretRef: + name: {{ include "apf.fullname" . }} + ports: + - name: http + containerPort: {{ .Values.app.container.port }} + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 4 + readinessProbe: + httpGet: + path: / + port: http + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 4 + resources: + {{- toYaml .Values.app.resources | nindent 12 }} diff --git a/helm/apf/templates/deployments/celery.yaml b/helm/apf/templates/deployments/celery.yaml new file mode 100644 index 00000000..222b90e1 --- /dev/null +++ b/helm/apf/templates/deployments/celery.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "apf.fullname" . }}-celery + labels: + {{- include "apf.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.celery.replicaCount }} + selector: + matchLabels: + {{- include "apf.selectorLabels" . | nindent 6 }}-celery + template: + metadata: + labels: + {{- include "apf.selectorLabels" . | nindent 8 }}-celery + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "apf.serviceAccountName" . }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.app.image }}:{{ .Values.app.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.celery.pullPolicy }} + command: + - invoke + - celery-queues + envFrom: + - secretRef: + name: {{ include "apf.fullname" . }} + resources: + {{- toYaml .Values.celery.resources | nindent 12 }} diff --git a/helm/apf/templates/ingress.yaml b/helm/apf/templates/ingress.yaml new file mode 100644 index 00000000..f6ae1cda --- /dev/null +++ b/helm/apf/templates/ingress.yaml @@ -0,0 +1,30 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "apf.fullname" . -}} +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "apf.labels" . | nindent 4 }} + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: web, websecure + cert-manager.io/cluster-issuer: {{ .Values.certificate.issuername | quote }} +spec: + {{- if .Values.ingress.tls }} + tls: + - secretName: {{ $fullName }}-cert + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: Prefix + backend: + serviceName: {{ $fullName }} + servicePort: {{ .servicePort }} + {{- end }} + {{- end }} + {{- end }} diff --git a/helm/apf/templates/secrets.yaml b/helm/apf/templates/secrets.yaml new file mode 100644 index 00000000..21118264 --- /dev/null +++ b/helm/apf/templates/secrets.yaml @@ -0,0 +1,46 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "apf.fullname" . }} + labels: + app: {{ template "apf.name" . }} + chart: {{ template "apf.chart" . }} + release: {{ .Release.Name | quote }} + heritage: {{ .Release.Service | quote }} +data: + BASE_URL: {{ .Values.baseUrl | b64enc | quote }} + BASE_EMAIL: {{ .Values.baseEmail | b64enc | quote }} + DEBUG: {{ .Values.app.debug | b64enc | quote }} + SILENCED_SYSTEM_CHECKS: {{ .Values.app.silencedSystemChecks | b64enc | quote }} + LOG_LEVEL: {{ .Values.app.log_level | b64enc | quote }} + SECRET_KEY: {{ required "secret key is needed" .Values.app.secretKey | b64enc | quote }} + # Postgres + POSTGRES_DB: {{ .Values.postgresql.postgresqlDatabase | b64enc | quote }} + POSTGRES_USER: {{ .Values.postgresql.postgresqlUsername | b64enc | quote }} + POSTGRES_HOST: {{ printf "%s-postgresql" (include "apf.fullname" .) | b64enc | quote }} + POSTGRES_PORT: {{ .Values.postgresql.service.port | b64enc | quote }} + POSTGRES_PASSWORD: {{ required "postgres password is needed" .Values.postgresql.postgresqlPassword | b64enc | quote }} + # Celery + CELERY_BROKER_PROTOCOL: {{ .Values.celery.broker.protocol | b64enc | quote }} + CELERY_BROKER_HOST: {{ printf "%s-redis-master" (include "apf.fullname" .) | b64enc | quote }} + CELERY_BROKER_PORT: {{ .Values.redis.redisPort | b64enc | quote }} + CELERY_BROKER_DB: {{ .Values.redis.db | b64enc | quote }} + CELERY_REDIRECT_STDOUTS_LEVEL: {{ .Values.celery.stdouts_level | b64enc | quote }} + # Telegram + TELEGRAM_TOKEN: {{ .Values.notifications.telegram.token | b64enc | quote }} + NOTIF_TELEGRAM_ENABLED: {{ .Values.notifications.telegram.enabled | b64enc | quote }} + TELEGRAM_CHAT_IDS: {{ .Values.notifications.telegram.chatIds | b64enc | quote }} + # Twitter + NOTIF_TWITTER_ENABLED: {{ .Values.notifications.twitter.enabled | b64enc | quote }} + TWITTER_API_KEY: {{ .Values.notifications.twitter.apiKey | b64enc | quote }} + TWITTER_API_SECRET: {{ .Values.notifications.twitter.apiSecret | b64enc | quote }} + TWITTER_ACCESS_TOKEN: {{ .Values.notifications.twitter.accessToken | b64enc | quote }} + TWITTER_ACCESS_TOKEN_SECRET: {{ .Values.notifications.twitter.accessTokenSecret | b64enc | quote }} + # Social Auth + SOCIAL_AUTH_GITHUB_KEY: {{ .Values.socialAuth.githubKey | b64enc | quote }} + SOCIAL_AUTH_GITHUB_SECRET: {{ .Values.socialAuth.githubSecret | b64enc | quote }} + # Recaptcha + RECAPTCHA_PUBLIC_KEY: {{ .Values.recaptcha.publicKey | b64enc | quote }} + RECAPTCHA_PRIVATE_KEY: {{ .Values.recaptcha.privateKey | b64enc | quote }} + # Sentry + SENTRY_URL: {{ .Values.sentry.dsn | b64enc | quote }} \ No newline at end of file diff --git a/helm/apf/templates/serviceaccount.yaml b/helm/apf/templates/serviceaccount.yaml new file mode 100644 index 00000000..f6a49916 --- /dev/null +++ b/helm/apf/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "apf.serviceAccountName" . }} + labels: + {{- include "apf.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/helm/apf/templates/services/app.yaml b/helm/apf/templates/services/app.yaml new file mode 100644 index 00000000..e1e3516b --- /dev/null +++ b/helm/apf/templates/services/app.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "apf.fullname" . }} + labels: + {{- include "apf.labels" . | nindent 4 }} +spec: + ports: + - port: {{ .Values.app.service.port }} + targetPort: {{ .Values.app.service.port }} + protocol: TCP + name: http + selector: + {{- include "apf.selectorLabels" . | nindent 4 }} diff --git a/helm/apf/templates/services/celery.yaml b/helm/apf/templates/services/celery.yaml new file mode 100644 index 00000000..27904536 --- /dev/null +++ b/helm/apf/templates/services/celery.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "apf.fullname" . }}-celery + labels: + {{- include "apf.labels" . | nindent 4 }}-celery +spec: + ports: + - port: {{ .Values.celery.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "apf.selectorLabels" . | nindent 4 }}-celery diff --git a/helm/apf/values.yaml b/helm/apf/values.yaml new file mode 100644 index 00000000..b0992e95 --- /dev/null +++ b/helm/apf/values.yaml @@ -0,0 +1,98 @@ +baseUrl: https://nemperfeina.cat +baseEmail: it@geekscat.org + +app: + image: geekscat/nem-per-feina + # Overrides the image tag whose default is the chart appVersion. + tag: latest + pullPolicy: Always + # workers is the amount of forked workers the server will have + workers: 4 + # threads is the amount of threads per worker + threads: 2 + debug: "True" + silencedSystemChecks: "[]" + log_level: INFO + secretKey: null + container: + port: 8000 + service: + port: 8000 + resources: {} + replicaCount: 1 + +postgresql: + serviceAccount: + enabled: true + postgresqlUsername: nemperfeina + postgresqlPassword: null + postgresqlDatabase: nemperfeina + service: + port: "5432" + +redis: + serviceAccount: + create: true + redisPort: "6379" + usePassword: false + db: "0" + +celery: + broker: + protocol: redis + pullPolicy: Always + service: + port: 8000 + resources: {} + stdouts_level: INFO + replicaCount: 1 + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +ingress: + enabled: true + app: + port: 80 + hosts: + - host: nemperfeina.cat + paths: + - path: / + servicePort: 8000 + +certificate: + hostname: nemperfeina.cat + issuername: letsencrypt-prod + +notifications: + telegram: + enabled: "False" + chatIds: "@chatID,@chatID2" + token: null + twitter: + enabled: "False" + apiKey: null + apiSecret: null + accessToken: null + accessTokenSecret: null + +socialAuth: + githubKey: null + githubSecret: null + +recaptcha: + publicKey: null + privateKey: null + +sentry: + dsn: null \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 094f5705..9755fa12 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,6 +46,7 @@ celery==5.0.1 python-telegram-bot==13.0.0 python-twitter==3.5 inclusive-django-range-fields==0.2.3 +uwsgi==2.0.19.1 django-cookie-law==2.0.3 django-recaptcha==2.0.6 -django-widget-tweaks==1.4.8 +django-widget-tweaks==1.4.8 \ No newline at end of file diff --git a/tasks.py b/tasks.py index 08e54b33..61b36223 100644 --- a/tasks.py +++ b/tasks.py @@ -52,8 +52,8 @@ def uwsgi( command_args = [ "uwsgi", - "--chdir=..", - "--module=bff.config.wsgi:application", + "--chdir=.", + "--module=jobs.wsgi:application", "--master", listen, f"--processes={workers}",