feat: Convert ansible to dockerfile

Dockerfile

@@ -1,112 +1,140 @@
-FROM ubuntu:focal as base
-MAINTAINER [email protected]
+FROM ubuntu:focal as app
 
-# Warning: This file is experimental.
-#
-# Short-term goals:
-# * Be a suitable replacement for the `edxops/credentials` image in devstack (in progress).
-# * Take advantage of Docker caching layers: aim to put commands in order of
-#   increasing cache-busting frequency.
-# * Related to ^, use no Ansible or Paver.
-# Long-term goal:
-# * Be a suitable base for production Credentials images. This may not yet be the case.
+# System requirements
 
-# Packages installed:
-# git; Used to pull in particular requirements from github rather than pypi,
-# and to check the sha of the code checkout.
+RUN apt-get update && \
+apt-get install -y software-properties-common && \
+apt-add-repository -y ppa:deadsnakes/ppa && apt-get update && \
+apt-get upgrade -qy && apt-get install language-pack-en locales git \
+python3.8-dev python3-virtualenv libmysqlclient-dev libssl-dev build-essential wget unzip -qy && \
+rm -rf /var/lib/apt/lists/*
 
-# language-pack-en locales; ubuntu locale support so that system utilities have a consistent
-# language and time zone.
+# Python is Python3.
+RUN ln -s /usr/bin/python3 /usr/bin/python
 
-# python; ubuntu doesnt ship with python, so this is the python we will use to run the application
+# Use UTF-8.
+RUN locale-gen en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
 
-# python3-pip; install pip to install application requirements.txt files
 
-# libssl-dev; # mysqlclient wont install without this.
+ARG COMMON_CFG_DIR="/edx/etc"
+ENV CREDENTIALS_CFG_DIR="${COMMON_CFG_DIR}/credentials"
 
-# libmysqlclient-dev; to install header files needed to use native C implementation for
-# MySQL-python for performance gains.
+ARG COMMON_APP_DIR="/edx/app"
+ARG CREDENTIALS_SERVICE_NAME="xxx"
+ARG CREDENTIALS_APP_DIR="${COMMON_APP_DIR}/credentials"
+ENV CREDENTIALS_APP_DIR="${COMMON_APP_DIR}/credentials"
+ENV SUPERVISOR_APP_DIR="${COMMON_APP_DIR}/supervisor"
+ENV CREDENTIALS_VENV_DIR="${COMMON_APP_DIR}/credentials/venvs/credentials"
+ARG SUPERVISOR_VENV_DIR="${SUPERVISOR_APP_DIR}/venvs/supervisor"
+ARG SUPERVISOR_AVAILABLE_DIR="${SUPERVISOR_APP_DIR}/conf.available.d"
+ARG SUPERVISOR_VENV_BIN="${SUPERVISOR_VENV_DIR}/bin"
+ARG SUPEVISOR_CTL="${SUPERVISOR_VENV_BIN}/supervisorctl"
+ARG SUPERVISOR_CFG_DIR="${SUPERVISOR_APP_DIR}/conf.d"
+ENV CREDENTIALS_CODE_DIR="${CREDENTIALS_APP_DIR}/credentials"
+ARG CREDENTIALS_NODEENV_DIR="${COMMON_APP_DIR}/credentials/nodeenvs/credentials"
+ARG CREDENTIALS_NODE_VERSION="16.14.0"
+ARG CREDENTIALS_NPM_VERSION="8.5.x"
+ARG SUPERVISOR_VERSION="4.2.1"
 
-# wget to download a watchman binary archive
+ENV PATH="$CREDENTIALS_VENV_DIR/bin:$PATH"
 
-# unzip to unzip a watchman binary archive
+ENV CREDENTIALS_NODEENV_DIR "${COMMON_APP_DIR}/credentials/nodeenvs/credentials"
+ENV CREDENTIALS_NODEENV_BIN "${CREDENTIALS_NODEENV_DIR}/bin"
+ENV CREDENTIALS_NODE_MODULES_DIR "${CREDENTIALS_CODE_DIR}}/node_modules"
+ENV CREDENTIALS_NODE_BIN "${CREDENTIALS_NODE_MODULES_DIR}/.bin"
 
-# If you add a package here please include a comment above describing what it is used for
-RUN apt-get update && \
-apt-get install -y software-properties-common && \
-apt-add-repository -y ppa:deadsnakes/ppa && apt-get update && \
-apt-get upgrade -qy && apt-get install language-pack-en locales git \
-python3.8-dev python3.8-venv libmysqlclient-dev libssl-dev build-essential wget unzip -qy && \
-rm -rf /var/lib/apt/lists/*
+RUN addgroup credentials
+RUN adduser --disabled-login --disabled-password credentials --ingroup credentials
+
+
+RUN mkdir -p "$CREDENTIALS_APP_DIR"
+
+# Working directory will be root of repo.
+WORKDIR ${CREDENTIALS_CODE_DIR}
+
+RUN virtualenv -p python3.8 --always-copy ${CREDENTIALS_VENV_DIR}
+RUN virtualenv -p python3.8 --always-copy ${SUPERVISOR_VENV_DIR}
 
-# Create Python env
-ENV VIRTUAL_ENV=/edx/app/credentials/venvs/credentials
-RUN python3.8 -m venv $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-# Create Node env
+ENV PATH "${CREDENTIALS_NODEENV_DIR}/bin:$PATH"
+# No need to activate credentials venv as it is already in path
 RUN pip install nodeenv
-ENV NODE_ENV=/edx/app/credentials/nodeenvs/credentials
-RUN nodeenv $NODE_ENV --node=16.14.0 --prebuilt
-ENV PATH="$NODE_ENV/bin:$PATH"
-RUN npm install -g [email protected]
 
-RUN locale-gen en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
-ENV DJANGO_SETTINGS_MODULE credentials.settings.production
+#install supervisor and deps in its virtualenv
+RUN . ${SUPERVISOR_VENV_BIN}/activate && \
+  pip install supervisor==${SUPERVISOR_VERSION} backoff==1.4.3 boto==2.48.0 && \
+  deactivate
 
+RUN nodeenv ${CREDENTIALS_NODEENV_DIR} --node=${CREDENTIALS_NODE_VERSION} --prebuilt
+RUN npm install -g npm@${CREDENTIALS_NPM_VERSION}
+
+# Copy just JS requirements and install them.
+COPY package.json package.json
+COPY package-lock.json package-lock.json
+RUN npm install --production
+
+# create supervisor job
+COPY /configuration_files/supervisor.conf /etc/systemd/system/supervisor.service
+COPY /configuration_files/supervisorctl ${SUPERVISOR_VENV_BIN}/supervisorctl
+
+# Copy just Python requirements & install them.
+COPY requirements ${CREDENTIALS_CODE_DIR}/requirements
+COPY Makefile ${CREDENTIALS_CODE_DIR}
+
+#Configurations from edx_service task
+RUN mkdir ${CREDENTIALS_APP_DIR}/data/
+RUN mkdir ${CREDENTIALS_APP_DIR}/staticfiles/
+RUN mkdir -p /edx/var/credentials/
+# Log dir
+RUN mkdir -p /edx/var/log/
+
+
+ENV CREDENTIALS_CFG="${COMMON_CFG_DIR}/credentials.yml"
+COPY configuration_files/credentials.yml ${CREDENTIALS_CFG}
+
+# credentials service config commands below
+RUN pip install -r ${CREDENTIALS_CODE_DIR}/requirements/production.txt
+
+# After the requirements so changes to the code will not bust the image cache
+COPY . ${CREDENTIALS_CODE_DIR}/
+
+COPY scripts/devstack.sh "$CREDENTIALS_APP_DIR/devstack.sh"
+# Enable supervisor script
+COPY scripts/credentials.sh $CREDENTIALS_APP_DIR/credentials.sh
+COPY /configuration_files/credentials.conf ${SUPERVISOR_AVAILABLE_DIR}/credentials.conf
+COPY /configuration_files/credentials.conf ${SUPERVISOR_CFG_DIR}/credentials.conf
+# Manage.py symlink
+COPY /manage.py /edx/bin/manage.credentials
+
+RUN chown credentials:credentials "$CREDENTIALS_APP_DIR/devstack.sh" && chmod a+x "$CREDENTIALS_APP_DIR/devstack.sh"
+
+# placeholder file for the time being unless devstack provisioning scripts need it.
+RUN touch ${CREDENTIALS_APP_DIR}/credentials_env
+# Expose ports.
 EXPOSE 18150
-RUN useradd -m --shell /bin/false app
-
-# Install watchman
-RUN wget https://github.com/facebook/watchman/releases/download/v2020.08.17.00/watchman-v2020.08.17.00-linux.zip
-RUN unzip watchman-v2020.08.17.00-linux.zip
-RUN mkdir -p /usr/local/{bin,lib} /usr/local/var/run/watchman
-RUN cp watchman-v2020.08.17.00-linux/bin/* /usr/local/bin
-RUN cp watchman-v2020.08.17.00-linux/lib/* /usr/local/lib
-RUN chmod 755 /usr/local/bin/watchman
-RUN chmod 2777 /usr/local/var/run/watchman
-
-# Now install credentials
-WORKDIR /edx/app/credentials/credentials
-
-# Copy the requirements explicitly even though we copy everything below
-# this prevents the image cache from busting unless the dependencies have changed.
-COPY requirements/production.txt /edx/app/credentials/credentials/requirements/production.txt
-COPY requirements/pip_tools.txt /edx/app/credentials/credentials/requirements/pip_tools.txt
-
-# Dependencies are installed as root so they cannot be modified by the application user.
-RUN pip install -r requirements/pip_tools.txt
-RUN pip install -r requirements/production.txt
-
-RUN mkdir -p /edx/var/log
-
-# Code is owned by root so it cannot be modified by the application user.
-# So we copy it before changing users.
-USER app
-
-# Gunicorn 19 does not log to stdout or stderr by default. Once we are past gunicorn 19, the logging to STDOUT need not be specified.
-CMD gunicorn --workers=2 --name credentials -c /edx/app/credentials/credentials/credentials/docker_gunicorn_configuration.py --log-file - --max-requests=1000 credentials.wsgi:application
-
-# This line is after the requirements so that changes to the code will not
-# bust the image cache
-COPY . /edx/app/credentials/credentials
-
-# We don't switch back to the app user for devstack because we need devstack users to be
-# able to update requirements and generally run things as root.
-FROM base as dev
-USER root
-ENV DJANGO_SETTINGS_MODULE credentials.settings.devstack
-RUN pip install -r /edx/app/credentials/credentials/requirements/dev.txt
 
-# Temporary compatibility hack while devstack is supporting
-# both the old `edxops/credentials` image and this image:
-# Add in a dummy ../credentials_env file.
-# The credentials_env file was originally needed for sourcing to get
-# environment variables like DJANGO_SETTINGS_MODULE, but now we just set
-# those variables right in the Dockerfile.
-RUN touch ../credentials_env
 
-CMD while true; do python ./manage.py runserver 0.0.0.0:18150; sleep 2; done
+FROM app as production
+
+ENV DJANGO_SETTINGS_MODULE credentials.settings.production
+
+COPY scripts/credentials.sh "$CREDENTIALS_APP_DIR/credentials.sh"
+
+#CMD ["gunicorn", "--workers=2", "--name", "credentials", "-c", "/edx/app/credentials/credentials/credentials/docker_gunicorn_configuration.py", "--log-file", "-", "--max-requests=1000", "credentials.wsgi:application"]
+#CMD ["sh", "-c", "gunicorn", "--workers=2", "--name", "credentials", "-c", "${CREDENTIALS_CODE_DIR}/docker_gunicorn_configuration.py", "--log-file", "-", "--max-requests=1000", "credentials.wsgi:application"]
+ENTRYPOINT ["/edx/app/credentials/credentials.sh"]
+
+
+FROM app as dev
+
+# credentials service config commands below
+RUN pip install -r ${CREDENTIALS_CODE_DIR}/requirements/dev.txt
+
+
+ENV DJANGO_SETTINGS_MODULE credentials.settings.devstack
+
+ENTRYPOINT ["/edx/app/credentials/devstack.sh"]
+CMD ["start"]

configuration_files/credentials.conf

@@ -0,0 +1,9 @@
+[program:credentials]
+
+command=/edx/app/credentials/credentials.sh
+user=www-data
+directory=/edx/app/credentials/credentials
+stdout_logfile=/edx/var/log/supervisor/%(program_name)s-stdout.log
+stderr_logfile=/edx/var/log/supervisor/%(program_name)s-stderr.log
+killasgroup=true
+stopasgroup=true

configuration_files/credentials.yml

@@ -0,0 +1,61 @@
+---
+
+API_ROOT: null
+BACKEND_SERVICE_EDX_OAUTH2_KEY: credentials-backend-service-key
+BACKEND_SERVICE_EDX_OAUTH2_PROVIDER_URL: http://localhost:18000/oauth2
+BACKEND_SERVICE_EDX_OAUTH2_SECRET: credentials-backend-service-secret
+CACHES:
+    default:
+        BACKEND: django.core.cache.backends.memcached.MemcachedCache
+        KEY_PREFIX: credentials
+        LOCATION:
+        - edx.devstack.memcached:11211
+CERTIFICATE_LANGUAGES:
+    en: English
+    es_419: Spanish
+CREDENTIALS_SERVICE_USER: credentials_service_user
+CSRF_COOKIE_SECURE: false
+DATABASES:
+    default:
+        ATOMIC_REQUESTS: false
+        CONN_MAX_AGE: 60
+        ENGINE: django.db.backends.mysql
+        HOST: edx.devstack.mysql
+        NAME: credentials
+        OPTIONS:
+            connect_timeout: 10
+            init_command: SET sql_mode='STRICT_TRANS_TABLES'
+        PASSWORD: password
+        PORT: '3306'
+        USER: credentials001
+EDX_DRF_EXTENSIONS:
+    OAUTH2_USER_INFO_URL: http://edx.devstack.lms:18000/oauth2/user_info
+EXTRA_APPS:
+- credentials.apps.edx_credentials_extensions
+FILE_STORAGE_BACKEND: {}
+JWT_AUTH:
+    JWT_AUTH_COOKIE_HEADER_PAYLOAD: edx-jwt-cookie-header-payload
+    JWT_AUTH_COOKIE_SIGNATURE: edx-jwt-cookie-signature
+    JWT_ISSUERS:
+    -   AUDIENCE: lms-key
+        ISSUER: http://localhost:18000/oauth2
+        SECRET_KEY: lms-secret
+    JWT_PUBLIC_SIGNING_JWK_SET: ''
+LANGUAGE_CODE: en
+LANGUAGE_COOKIE_NAME: openedx-language-preference
+MEDIA_STORAGE_BACKEND:
+    DEFAULT_FILE_STORAGE: django.core.files.storage.FileSystemStorage
+    MEDIA_ROOT: /edx/var/credentials/media
+    MEDIA_URL: /media/
+SECRET_KEY: SET-ME-TO-A-UNIQUE-LONG-RANDOM-STRING
+SESSION_EXPIRE_AT_BROWSER_CLOSE: false
+SOCIAL_AUTH_EDX_OAUTH2_ISSUER: http://127.0.0.1:8000
+SOCIAL_AUTH_EDX_OAUTH2_KEY: credentials-sso-key
+SOCIAL_AUTH_EDX_OAUTH2_LOGOUT_URL: http://localhost:18000/logout
+SOCIAL_AUTH_EDX_OAUTH2_SECRET: credentials-sso-secret
+SOCIAL_AUTH_EDX_OAUTH2_URL_ROOT: http://127.0.0.1:8000
+SOCIAL_AUTH_REDIRECT_IS_HTTPS: false
+STATICFILES_STORAGE: django.contrib.staticfiles.storage.ManifestStaticFilesStorage
+STATIC_ROOT: /edx/var/credentials/staticfiles
+TIME_ZONE: UTC
+USERNAME_REPLACEMENT_WORKER: OVERRIDE THIS WITH A VALID USERNAME

configuration_files/supervisor.conf

@@ -0,0 +1,29 @@
+[Unit]
+Description=supervisord - Supervisor process control system
+Documentation=http://supervisord.org
+After=network.target
+
+
+[Service]
+
+# User will be applied only to ExecStart, not other commands (i.e. ExecStartPre)
+# This is needed because pre_supervisor needs to write to supervisor/conf.d, which
+# supervisor_service_user does not have permission to do.
+PermissionsStartOnly=true
+User=www-data
+
+Type=forking
+TimeoutSec=432000
+
+ExecStart=/edx/app/supervisor/venvs/supervisor/bin/supervisord --configuration /edx/app/supervisor/supervisord.conf
+ExecReload=/edx/app/supervisor/venvs/supervisor/bin/supervisorctl reload
+ExecStop=/edx/app/supervisor/venvs/supervisor/bin/supervisorctl shutdown
+
+# Trust supervisor to kill all its children
+# Otherwise systemd will see that ExecStop ^ comes back synchronously and say "Oh, I can kill everyone in this cgroup"
+# https://www.freedesktop.org/software/systemd/man/systemd.service.html#ExecStop=
+# https://www.freedesktop.org/software/systemd/man/systemd.kill.html
+KillMode=none
+
+[Install]
+WantedBy=multi-user.target

configuration_files/supervisorctl

@@ -0,0 +1,8 @@
+#!/edx/app/supervisor/venvs/supervisor/bin/python
+# -*- coding: utf-8 -*-
+import re
+import sys
+from supervisor.supervisorctl import main
+if __name__ == '__main__':
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

credentials_gunicorn.py

@@ -0,0 +1,18 @@
+"""
+gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html
+Ansible managed
+"""
+
+timeout = 300
+bind = "127.0.0.1:8150"
+pythonpath = "/edx/app/credentials/credentials"
+workers = 2
+worker_class = "gevent"
+
+limit_request_field_size = 16384
+
+
+
+
+def pre_request(worker, req):
+    worker.log.info("%s %s" % (req.method, req.path))

scripts/credentials.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# Ansible managed
+
+
+
+export EDX_REST_API_CLIENT_NAME="default_env-default_deployment-credentials"
+
+source /edx/app/credentials/credentials_env
+
+exec /edx/app/credentials/venvs/credentials/bin/gunicorn -c /edx/app/credentials/credentials_gunicorn.py --reload credentials.wsgi:application