From 8fefa9309f61aa3550adb72590c3b68f029a0488 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 23 Jul 2020 12:00:52 -0400 Subject: [PATCH 1/9] Use a regular (non-root) user --- Dockerfile | 144 +++++++++++--------- docker/init.bash | 29 +--- docker/run_celery.bash | 4 +- docker/run_celery_beat.bash | 4 +- docker/run_uwsgi.bash | 22 +-- docker/setup_cron.bash | 15 ++ docker/setup_pydev_debugger.bash | 9 ++ docker/{sync_static.sh => sync_static.bash} | 8 +- 8 files changed, 130 insertions(+), 105 deletions(-) create mode 100755 docker/setup_cron.bash create mode 100755 docker/setup_pydev_debugger.bash rename docker/{sync_static.sh => sync_static.bash} (79%) diff --git a/Dockerfile b/Dockerfile index ca874fff6..662453dee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,45 +1,59 @@ FROM nikolaik/python-nodejs:python3.8-nodejs10 +ARG DOCKER_USER_PASSWORD=kobo + +RUN if [ "$DOCKER_USER_PASSWORD" = "kobo" ]; then \ + echo "###################################################################" && \ + echo "# You are using default password: \`kobo\`. #" && \ + echo "# You should build your image with: #" && \ + echo "# \`docker build --build-arg DOCKER_USER_PASSWORD=\` #" && \ + echo "###################################################################" && \ + sleep 5; \ + fi; + +# Declare environment variables ENV DEBIAN_FRONTEND=noninteractive ENV LANG=en_US.UTF-8 ENV LANGUAGE=en_US:en ENV LC_ALL=en_US.UTF-8 -ENV VIRTUAL_ENV=/opt/venv - -ENV KOBOCAT_LOGS_DIR=/srv/logs \ - DJANGO_SETTINGS_MODULE=kobo.settings.prod \ +ENV VIRTUAL_ENV=/opt/venv \ + KOBOCAT_LOGS_DIR=/srv/logs \ + DJANGO_SETTINGS_MODULE=onadata.settings.kc_environ \ # The mountpoint of a volume shared with the `nginx` container. Static files will - # be copied there. + # be copied there. NGINX_STATIC_DIR=/srv/static \ KOBOCAT_SRC_DIR=/srv/src/kobocat \ BACKUPS_DIR=/srv/backups \ - TMP_PATH=/srv/tmp \ - INIT_PATH=/srv/init + TMP_DIR=/srv/tmp \ + UWSGI_USER=kobo \ + UWSGI_GROUP=kobo \ + SERVICES_DIR=/etc/runit/runsvdir/default \ + CELERY_PID_DIR=/var/run/celery -# Install Dockerize. +# Install Dockerize ENV DOCKERIZE_VERSION v0.6.1 RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz -P /tmp \ && tar -C /usr/local/bin -xzvf /tmp/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ && rm /tmp/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz -########################################## -# Create build directories # -########################################## - -RUN mkdir -p "${NGINX_STATIC_DIR}" && \ - mkdir -p "${KOBOCAT_SRC_DIR}" && \ - mkdir -p "${TMP_PATH}" && \ - mkdir -p "${BACKUPS_DIR}" && \ - mkdir -p "${INIT_PATH}" - -########################################## -# Install `apt` packages. # -########################################## +# Create needed directories +RUN mkdir -p ${NGINX_STATIC_DIR} && \ + mkdir -p ${KOBOCAT_SRC_DIR} && \ + mkdir -p ${TMP_DIR} && \ + mkdir -p ${BACKUPS_DIR} && \ + mkdir -p ${CELERY_PID_DIR} && \ + mkdir -p ${SERVICES_DIR}/uwsgi/supervise && \ + mkdir -p ${SERVICES_DIR}/celery/supervise && \ + mkdir -p ${SERVICES_DIR}/celery_beat/supervise && \ + mkdir -p ${KOBOCAT_LOGS_DIR}/ && \ + mkdir -p ${KOBOCAT_SRC_DIR}/emails + +# Install `apt` packages. RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - -RUN apt -qq update && \ - apt -qq -y install \ +RUN apt-get -qq update && \ + apt-get -qq -y install \ gdal-bin \ libproj-dev \ gettext \ @@ -47,63 +61,67 @@ RUN apt -qq update && \ locales \ runit-init \ rsync \ - vim && \ - apt clean && \ + vim \ + sudo \ + cron && \ + apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -########################### -# Install locales # -########################### +# Install locales +RUN echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen && \ + locale-gen && dpkg-reconfigure locales -f noninteractive -RUN echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen -RUN locale-gen && dpkg-reconfigure locales -f noninteractive +# Create local user UWSGI_USER` with sudo privileges +RUN adduser --disabled-password --gecos '' "$UWSGI_USER" && \ + echo "${UWSGI_USER}:${DOCKER_USER_PASSWORD}" | chpasswd && \ + adduser "$UWSGI_USER" sudo -########################### -# Copy KoBoCAT directory # -########################### +# Add command to sudoers to avoid asking for `UWSGI_USER`'s password# +RUN echo "Cmnd_Alias SETUP_CRON = ${KOBOCAT_SRC_DIR}/docker/setup_cron.bash" >> /etc/sudoers && \ + echo "Cmnd_Alias SETUP_PYDEV_DEBUGGER = ${KOBOCAT_SRC_DIR}/docker/setup_pydev_debugger.bash" >> /etc/sudoers && \ + echo "Cmnd_Alias SYNC_STATIC = ${KOBOCAT_SRC_DIR}/docker/sync_static.bash" >> /etc/sudoers && \ + echo "$UWSGI_USER ALL=(ALL) NOPASSWD:SETENV: SETUP_CRON" >> /etc/sudoers && \ + echo "$UWSGI_USER ALL=(ALL) NOPASSWD:SETENV: SETUP_PYDEV_DEBUGGER" >> /etc/sudoers && \ + echo "$UWSGI_USER ALL=(ALL) NOPASSWD:SETENV: SYNC_STATIC" >> /etc/sudoers +# Copy KoBoCAT directory COPY . "${KOBOCAT_SRC_DIR}" -########################### -# Install `pip` packages. # -########################### - +# Install `pip` packages RUN virtualenv "$VIRTUAL_ENV" ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN pip install --quiet --upgrade pip && \ pip install --quiet pip-tools -COPY ./dependencies/pip/prod.txt /srv/tmp/pip_dependencies.txt -RUN pip-sync /srv/tmp/pip_dependencies.txt 1>/dev/null && \ +COPY ./dependencies/pip/prod.txt "${TMP_DIR}/pip_dependencies.txt" +RUN pip-sync "${TMP_DIR}/pip_dependencies.txt" 1>/dev/null && \ rm -rf ~/.cache/pip -########################################## -# Persist the log and email directories. # -########################################## - -RUN mkdir -p "${KOBOCAT_LOGS_DIR}/" "${KOBOCAT_SRC_DIR}/emails" && \ - chown -R "${UWSGI_USER}" "${KOBOCAT_SRC_DIR}/emails/" && \ - chown -R "${UWSGI_USER}" "${KOBOCAT_LOGS_DIR}" - -################################################# -# Handle runtime tasks and create main process. # -################################################# - # Using `/etc/profile.d/` as a repository for non-hard-coded environment variable overrides. -RUN echo "export PATH=${PATH}" >> /etc/profile -RUN echo 'source /etc/profile' >> /root/.bashrc - -# Prepare for execution. -RUN mkdir -p /etc/service/uwsgi && \ - # Remove getty* services - rm -rf /etc/runit/runsvdir/default/getty-tty* && \ - cp "${KOBOCAT_SRC_DIR}/docker/run_uwsgi.bash" /etc/service/uwsgi/run && \ - mkdir -p /etc/service/celery && \ - ln -s "${KOBOCAT_SRC_DIR}/docker/run_celery.bash" /etc/service/celery/run && \ - mkdir -p /etc/service/celery_beat && \ - ln -s "${KOBOCAT_SRC_DIR}/docker/run_celery_beat.bash" /etc/service/celery_beat/run && \ +RUN echo "export PATH=${PATH}" >> /etc/profile && \ + echo 'source /etc/profile' >> /root/.bashrc + +# Remove getty* services to avoid errors at start-up +RUN rm -rf ${SERVICES_DIR}/getty-tty* + +# Create symlinks for runsv services +RUN ln -s ${KOBOCAT_SRC_DIR}/docker/run_uwsgi.bash ${SERVICES_DIR}/uwsgi/run && \ + ln -s ${KOBOCAT_SRC_DIR}/docker/run_celery.bash ${SERVICES_DIR}/celery/run && \ + ln -s ${KOBOCAT_SRC_DIR}/docker/run_celery_beat.bash ${SERVICES_DIR}/celery_beat/run + +# Add/Restore `UWSGI_USER`'s permissions +RUN chown -R ${UWSGI_USER} ${SERVICES_DIR} && \ + chown -R ":${UWSGI_GROUP}" ${CELERY_PID_DIR} && \ + chmod g+w ${CELERY_PID_DIR} && \ + chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${KOBOCAT_SRC_DIR}/emails/ && \ + chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${KOBOCAT_LOGS_DIR} && \ + chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${TMP_DIR} && \ + chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${VIRTUAL_ENV} WORKDIR "${KOBOCAT_SRC_DIR}" EXPOSE 8000 +# Run container with `UWSGI_USER` +USER $UWSGI_USER + CMD ["/bin/bash", "-c", "exec ${KOBOCAT_SRC_DIR}/docker/init.bash"] diff --git a/docker/init.bash b/docker/init.bash index 48e903295..4371bf80c 100755 --- a/docker/init.bash +++ b/docker/init.bash @@ -5,6 +5,8 @@ source /etc/profile echo 'KoBoCAT intializing...' +echo "WHO AM I INIT?: $(whoami)" + cd "${KOBOCAT_SRC_DIR}" if [[ -z $DATABASE_URL ]]; then @@ -16,30 +18,13 @@ fi echo 'Running migrations...' python manage.py migrate --noinput -rm -f /etc/cron.d/clean_up_tmp -cp docker/cron/clean_up_tmp /etc/cron.d/ -echo 'KoBoCat tmp clean-up cron installed' - -rm -f /etc/cron.d/backup_media_crontab -if [[ -z "${KOBOCAT_MEDIA_BACKUP_SCHEDULE}" ]]; then - echo 'KoBoCAT media automatic backups disabled.' -else - # Should we first validate the schedule e.g. with `chkcrontab`? - cat "${KOBOCAT_SRC_DIR}/docker/backup_media_crontab.envsubst" | envsubst > /etc/cron.d/backup_media_crontab - echo "KoBoCAT media automatic backup schedule: ${KOBOCAT_MEDIA_BACKUP_SCHEDULE}" -fi - -/bin/bash ${KOBOCAT_SRC_DIR}/docker/sync_static.sh - -# Keep it as is, not tested with Python3 -rm -rf /etc/profile.d/pydev_debugger.bash.sh -if [[ -d /srv/pydev_orig && -n "${KOBOCAT_PATH_FROM_ECLIPSE_TO_PYTHON_PAIRS}" ]]; then - echo 'Enabling PyDev remote debugging.' - "${KOBOCAT_SRC_DIR}/docker/setup_pydev.bash" -fi +echo 'Setting up cron tasks...' +sudo -E ${KOBOCAT_SRC_DIR}/docker/setup_cron.bash +sudo -E ${KOBOCAT_SRC_DIR}/docker/setup_pydev_debugger.bash +sudo -E ${KOBOCAT_SRC_DIR}/docker/sync_static.bash echo 'Cleaning up Celery PIDs...' -rm -rf /tmp/celery*.pid +rm -rf ${CELERY_PID_DIR}/*.pid echo 'KoBoCAT initialization complete.' diff --git a/docker/run_celery.bash b/docker/run_celery.bash index d043a82a9..5af2966cb 100755 --- a/docker/run_celery.bash +++ b/docker/run_celery.bash @@ -4,7 +4,7 @@ source /etc/profile # Run the main Celery worker cd "${KOBOCAT_SRC_DIR}" -exec /sbin/setuser "${UWSGI_USER}" celery worker -A onadata -Ofair --loglevel=info \ +exec celery worker -A onadata -Ofair --loglevel=info \ --hostname=kobocat_main_worker@%h \ --logfile=${KOBOCAT_LOGS_DIR}/celery.log \ - --pidfile=/tmp/celery.pid + --pidfile=${CELERY_PID_DIR}/celery.pid diff --git a/docker/run_celery_beat.bash b/docker/run_celery_beat.bash index e777dff6c..19d522f16 100755 --- a/docker/run_celery_beat.bash +++ b/docker/run_celery_beat.bash @@ -4,7 +4,7 @@ source /etc/profile # Run the main Celery beat worker cd "${KOBOCAT_SRC_DIR}" -exec /sbin/setuser "${UWSGI_USER}" celery beat -A onadata --loglevel=info \ +exec celery beat -A onadata --loglevel=info \ --logfile=${KOBOCAT_LOGS_DIR}/celery_beat.log \ - --pidfile=/tmp/celery_beat.pid \ + --pidfile=${CELERY_PID_DIR}/celery_beat.pid \ --scheduler django_celery_beat.schedulers:DatabaseScheduler diff --git a/docker/run_uwsgi.bash b/docker/run_uwsgi.bash index 76f591dae..2682afd8b 100755 --- a/docker/run_uwsgi.bash +++ b/docker/run_uwsgi.bash @@ -1,34 +1,33 @@ #!/usr/bin/env bash -set -e source /etc/profile - if [[ "$(stat -c '%U' ${KOBOCAT_LOGS_DIR})" != "${UWSGI_USER}" ]]; then echo 'Restoring ownership of Logs directory.' chown -R "${UWSGI_USER}" "${KOBOCAT_LOGS_DIR}" fi KOBOCAT_WEB_SERVER="${KOBOCAT_WEB_SERVER:-uWSGI}" -uwsgi_command="/sbin/setuser ${UWSGI_USER} $(command -v uwsgi) --ini ${KOBOCAT_SRC_DIR}/docker/kobocat.ini" + +echo "WHO AM I UWSGI?: $(whoami)" if [[ "${KOBOCAT_WEB_SERVER,,}" == "uwsgi" ]]; then cd "${KOBOCAT_SRC_DIR}" - DIFF=$(diff "${KOBOCAT_SRC_DIR}/dependencies/pip/prod.txt" "/srv/tmp/pip_dependencies.txt") + DIFF=$(diff "${KOBOCAT_SRC_DIR}/dependencies/pip/prod.txt" "${TMP_DIR}/pip_dependencies.txt") if [[ -n "$DIFF" ]]; then echo "Syncing pip dependencies..." pip-sync dependencies/pip/prod.txt 1>/dev/null - cp "dependencies/pip/prod.txt" "/srv/tmp/pip_dependencies.txt" + cp "dependencies/pip/prod.txt" "${TMP_DIR}/pip_dependencies.txt" fi - echo 'Running `kobocat` container with uWSGI application server.' - exec ${uwsgi_command} + echo "Running \`KoBoCAT\` container with uWSGI application server." + UWSGI_COMMAND="$(command -v uwsgi) --ini ${KOBOCAT_SRC_DIR}/docker/kobocat.ini" else cd "${KOBOCAT_SRC_DIR}" - DIFF=$(diff "${KOBOCAT_SRC_DIR}/dependencies/pip/dev.txt" "/srv/tmp/pip_dependencies.txt") + DIFF=$(diff "${KOBOCAT_SRC_DIR}/dependencies/pip/dev.txt" "${TMP_DIR}/pip_dependencies.txt") if [[ -n "$DIFF" ]]; then echo "Syncing pip dependencies..." pip-sync dependencies/pip/dev.txt 1>/dev/null - cp "dependencies/pip/dev.txt" "/srv/tmp/pip_dependencies.txt" + cp "dependencies/pip/dev.txt" "${TMP_DIR}/pip_dependencies.txt" fi if [[ -n "$RAVEN_DSN" ]]; then @@ -36,6 +35,7 @@ else pip install raven fi - echo 'Running `kobocat` container with `runserver_plus` debugging application server.' - exec python manage.py runserver_plus 0:8000 + echo "Running KoBoCAT container with \`runserver_plus\` debugging application server." + UWSGI_COMMAND="python manage.py runserver_plus 0:8000" fi +exec ${UWSGI_COMMAND} diff --git a/docker/setup_cron.bash b/docker/setup_cron.bash new file mode 100755 index 000000000..9ad6556fb --- /dev/null +++ b/docker/setup_cron.bash @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + +rm -f /etc/cron.d/clean_up_tmp +cp docker/cron/clean_up_tmp /etc/cron.d/ +echo 'KoBoCAT tmp clean-up cron installed' + +rm -f /etc/cron.d/backup_media_crontab +if [[ -z "${KOBOCAT_MEDIA_BACKUP_SCHEDULE}" ]]; then + echo 'KoBoCAT media automatic backups disabled.' +else + # Should we first validate the schedule e.g. with `chkcrontab`? + cat "${KOBOCAT_SRC_DIR}/docker/backup_media_crontab.envsubst" | envsubst > /etc/cron.d/backup_media_crontab + echo "KoBoCAT media automatic backup schedule: ${KOBOCAT_MEDIA_BACKUP_SCHEDULE}" +fi diff --git a/docker/setup_pydev_debugger.bash b/docker/setup_pydev_debugger.bash new file mode 100755 index 000000000..c8f7dfc50 --- /dev/null +++ b/docker/setup_pydev_debugger.bash @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +# Keep it as is, not tested with Python3 +rm -rf /etc/profile.d/pydev_debugger.bash.sh +if [[ -d /srv/pydev_orig && -n "${KOBOCAT_PATH_FROM_ECLIPSE_TO_PYTHON_PAIRS}" ]]; then + echo 'Enabling PyDev remote debugging.' + "${KOBOCAT_SRC_DIR}/docker/setup_pydev.bash" +fi diff --git a/docker/sync_static.sh b/docker/sync_static.bash similarity index 79% rename from docker/sync_static.sh rename to docker/sync_static.bash index c752e0e8d..e1d963bd3 100755 --- a/docker/sync_static.sh +++ b/docker/sync_static.bash @@ -1,12 +1,10 @@ #!/bin/bash set -e -source /etc/profile - mkdir -p "${KOBOCAT_SRC_DIR}/onadata/static" echo "Collecting static files..." -python manage.py collectstatic -v 0 --noinput +${VIRTUAL_ENV}/bin/python manage.py collectstatic -v 0 --noinput echo "Done" # `chown -R` becomes very slow once a fair amount of media has been collected, @@ -18,10 +16,10 @@ echo '%%%%%%% NOTICE %%%%%%%' echo '% To avoid long delays, we no longer reset ownership *recursively*' echo '% every time this container starts. If you have trouble with' echo '% permissions, please run the following command inside the' -echo '% `kobocat` container:' +echo '% KoBoCAT container:' echo "% chown -R \"${UWSGI_USER}\" \"${KOBOCAT_SRC_DIR}\"" echo '%%%%%%%%%%%%%%%%%%%%%%' echo "Syncing to nginx folder..." -rsync -aq --delete --chown=www-data "${KPI_SRC_DIR}/onadata/static/" "${NGINX_STATIC_DIR}/" +rsync -aq --delete --chown=www-data "${KOBOCAT_SRC_DIR}/onadata/static/" "${NGINX_STATIC_DIR}/" echo "Done" From a0c2734f35a0f271b385315c31560fb9cda41ccf Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 23 Jul 2020 15:01:24 -0400 Subject: [PATCH 2/9] Improve docker image build --- Dockerfile | 17 +++++++++-------- docker/init.bash | 4 +--- docker/run_uwsgi.bash | 18 ++++++++++++++++-- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 662453dee..3c00cd644 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ ENV VIRTUAL_ENV=/opt/venv \ TMP_DIR=/srv/tmp \ UWSGI_USER=kobo \ UWSGI_GROUP=kobo \ - SERVICES_DIR=/etc/runit/runsvdir/default \ + SERVICES_DIR=/srv/services \ CELERY_PID_DIR=/var/run/celery # Install Dockerize @@ -43,9 +43,9 @@ RUN mkdir -p ${NGINX_STATIC_DIR} && \ mkdir -p ${TMP_DIR} && \ mkdir -p ${BACKUPS_DIR} && \ mkdir -p ${CELERY_PID_DIR} && \ - mkdir -p ${SERVICES_DIR}/uwsgi/supervise && \ - mkdir -p ${SERVICES_DIR}/celery/supervise && \ - mkdir -p ${SERVICES_DIR}/celery_beat/supervise && \ + mkdir -p ${SERVICES_DIR}/uwsgi && \ + mkdir -p ${SERVICES_DIR}/celery && \ + mkdir -p ${SERVICES_DIR}/celery_beat && \ mkdir -p ${KOBOCAT_LOGS_DIR}/ && \ mkdir -p ${KOBOCAT_SRC_DIR}/emails @@ -58,6 +58,7 @@ RUN apt-get -qq update && \ libproj-dev \ gettext \ postgresql-client \ + openjdk-11-jre \ locales \ runit-init \ rsync \ @@ -100,8 +101,8 @@ RUN pip-sync "${TMP_DIR}/pip_dependencies.txt" 1>/dev/null && \ RUN echo "export PATH=${PATH}" >> /etc/profile && \ echo 'source /etc/profile' >> /root/.bashrc -# Remove getty* services to avoid errors at start-up -RUN rm -rf ${SERVICES_DIR}/getty-tty* +# Remove getty* services to avoid errors of absent tty at sv start-up +RUN rm -rf /etc/runit/runsvdir/default/getty-tty* # Create symlinks for runsv services RUN ln -s ${KOBOCAT_SRC_DIR}/docker/run_uwsgi.bash ${SERVICES_DIR}/uwsgi/run && \ @@ -109,9 +110,9 @@ RUN ln -s ${KOBOCAT_SRC_DIR}/docker/run_uwsgi.bash ${SERVICES_DIR}/uwsgi/run && ln -s ${KOBOCAT_SRC_DIR}/docker/run_celery_beat.bash ${SERVICES_DIR}/celery_beat/run # Add/Restore `UWSGI_USER`'s permissions -RUN chown -R ${UWSGI_USER} ${SERVICES_DIR} && \ - chown -R ":${UWSGI_GROUP}" ${CELERY_PID_DIR} && \ +RUN chown -R ":${UWSGI_GROUP}" ${CELERY_PID_DIR} && \ chmod g+w ${CELERY_PID_DIR} && \ + chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${SERVICES_DIR} && \ chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${KOBOCAT_SRC_DIR}/emails/ && \ chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${KOBOCAT_LOGS_DIR} && \ chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${TMP_DIR} && \ diff --git a/docker/init.bash b/docker/init.bash index 4371bf80c..4f49a3158 100755 --- a/docker/init.bash +++ b/docker/init.bash @@ -5,8 +5,6 @@ source /etc/profile echo 'KoBoCAT intializing...' -echo "WHO AM I INIT?: $(whoami)" - cd "${KOBOCAT_SRC_DIR}" if [[ -z $DATABASE_URL ]]; then @@ -28,4 +26,4 @@ rm -rf ${CELERY_PID_DIR}/*.pid echo 'KoBoCAT initialization complete.' -exec /usr/bin/runsvdir /etc/service +exec /usr/bin/runsvdir "${SERVICES_DIR}" diff --git a/docker/run_uwsgi.bash b/docker/run_uwsgi.bash index 2682afd8b..85f4090c6 100755 --- a/docker/run_uwsgi.bash +++ b/docker/run_uwsgi.bash @@ -9,15 +9,21 @@ fi KOBOCAT_WEB_SERVER="${KOBOCAT_WEB_SERVER:-uWSGI}" -echo "WHO AM I UWSGI?: $(whoami)" - if [[ "${KOBOCAT_WEB_SERVER,,}" == "uwsgi" ]]; then cd "${KOBOCAT_SRC_DIR}" DIFF=$(diff "${KOBOCAT_SRC_DIR}/dependencies/pip/prod.txt" "${TMP_DIR}/pip_dependencies.txt") if [[ -n "$DIFF" ]]; then + # Celery services need to be stopped to avoid raising errors during + # pip-sync + echo "Stopping Celery services..." + sv stop "${SERVICES_DIR}/celery" + sv stop "${SERVICES_DIR}/celery_beat" echo "Syncing pip dependencies..." pip-sync dependencies/pip/prod.txt 1>/dev/null cp "dependencies/pip/prod.txt" "${TMP_DIR}/pip_dependencies.txt" + echo "Restarting Celery..." + sv start "${SERVICES_DIR}/celery" + sv start "${SERVICES_DIR}/celery_beat" fi echo "Running \`KoBoCAT\` container with uWSGI application server." UWSGI_COMMAND="$(command -v uwsgi) --ini ${KOBOCAT_SRC_DIR}/docker/kobocat.ini" @@ -25,9 +31,17 @@ else cd "${KOBOCAT_SRC_DIR}" DIFF=$(diff "${KOBOCAT_SRC_DIR}/dependencies/pip/dev.txt" "${TMP_DIR}/pip_dependencies.txt") if [[ -n "$DIFF" ]]; then + # Celery services need to be stopped to avoid raising errors during + # pip-sync + echo "Stopping Celery services..." + sv stop "${SERVICES_DIR}/celery" + sv stop "${SERVICES_DIR}/celery_beat" echo "Syncing pip dependencies..." pip-sync dependencies/pip/dev.txt 1>/dev/null cp "dependencies/pip/dev.txt" "${TMP_DIR}/pip_dependencies.txt" + echo "Restarting Celery..." + sv start "${SERVICES_DIR}/celery" + sv start "${SERVICES_DIR}/celery_beat" fi if [[ -n "$RAVEN_DSN" ]]; then From b6cf3f6117c2b8281cb689eedd379355bac14f8f Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 23 Jul 2020 16:42:54 -0400 Subject: [PATCH 3/9] fixed wrong settings file name in Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3c00cd644..509a54367 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ ENV LC_ALL=en_US.UTF-8 ENV VIRTUAL_ENV=/opt/venv \ KOBOCAT_LOGS_DIR=/srv/logs \ - DJANGO_SETTINGS_MODULE=onadata.settings.kc_environ \ + DJANGO_SETTINGS_MODULE=onadata.settings.prod \ # The mountpoint of a volume shared with the `nginx` container. Static files will # be copied there. NGINX_STATIC_DIR=/srv/static \ From bb0a0e06950bd32c80cf340d3f3913140d329453 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 28 Jul 2020 15:34:03 -0400 Subject: [PATCH 4/9] update: pip dependencies --- dependencies/pip/dev.txt | 7 +++---- dependencies/pip/prod.txt | 10 +++++----- dependencies/pip/requirements.txt | 7 +++---- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/dependencies/pip/dev.txt b/dependencies/pip/dev.txt index a9853a4cf..833545c01 100644 --- a/dependencies/pip/dev.txt +++ b/dependencies/pip/dev.txt @@ -36,10 +36,11 @@ django-redis-sessions==0.6.1 # via -r dependencies/pip/requirements.in django-registration-redux==2.8 # via -r dependencies/pip/requirements.in django-render-block==0.7 # via django-templated-email django-reversion==3.0.1 # via -r dependencies/pip/requirements.in +django-storages==1.9.1 # via -r dependencies/pip/requirements.in django-taggit==1.3.0 # via -r dependencies/pip/requirements.in django-templated-email==2.3.0 # via -r dependencies/pip/requirements.in django-timezone-field==4.0 # via -r dependencies/pip/requirements.in, django-celery-beat -django==2.2.14 # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield +django==2.2.14 # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-storages, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield djangorestframework-csv==2.1.0 # via -r dependencies/pip/requirements.in djangorestframework-guardian==0.3.0 # via -r dependencies/pip/requirements.in djangorestframework-jsonp==1.0.2 # via -r dependencies/pip/requirements.in @@ -60,7 +61,6 @@ gitpython==3.1.7 # via transifex-client httmock==1.3.0 # via -r dependencies/pip/dev.in httplib2==0.18.1 # via -r dependencies/pip/requirements.in idna==2.10 # via requests -importlib-metadata==1.7.0 # via jsonschema, kombu, markdown, path, pluggy, pytest ipdb==0.13.3 # via -r dependencies/pip/dev.in ipython-genutils==0.2.0 # via traitlets ipython==7.16.1 # via -r dependencies/pip/dev.in, ipdb @@ -83,7 +83,7 @@ openpyxl==3.0.4 # via -r dependencies/pip/requirements.in packaging==20.4 # via pytest pandas==1.0.5 # via -r dependencies/pip/requirements.in parso==0.7.0 # via jedi -path.py==12.4.0 # via -r dependencies/pip/requirements.in, formpack +path.py==12.4.0 # via formpack path==13.1.0 # via path.py pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython @@ -131,7 +131,6 @@ werkzeug==1.0.1 # via -r dependencies/pip/dev.in xlrd==1.2.0 # via -r dependencies/pip/requirements.in, pyxform xlsxwriter==1.2.9 # via formpack xlwt==1.3.0 # via -r dependencies/pip/requirements.in -zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/dependencies/pip/prod.txt b/dependencies/pip/prod.txt index 62febf9f1..e13354f6c 100644 --- a/dependencies/pip/prod.txt +++ b/dependencies/pip/prod.txt @@ -34,10 +34,11 @@ django-redis-sessions==0.6.1 # via -r dependencies/pip/requirements.in django-registration-redux==2.8 # via -r dependencies/pip/requirements.in django-render-block==0.7 # via django-templated-email django-reversion==3.0.1 # via -r dependencies/pip/requirements.in +django-storages==1.9.1 # via -r dependencies/pip/requirements.in django-taggit==1.3.0 # via -r dependencies/pip/requirements.in django-templated-email==2.3.0 # via -r dependencies/pip/requirements.in django-timezone-field==4.0 # via -r dependencies/pip/requirements.in, django-celery-beat -django==2.2.14 # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield +django==2.2.14 # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-storages, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield djangorestframework-csv==2.1.0 # via -r dependencies/pip/requirements.in djangorestframework-guardian==0.3.0 # via -r dependencies/pip/requirements.in djangorestframework-jsonp==1.0.2 # via -r dependencies/pip/requirements.in @@ -57,7 +58,6 @@ gitdb==4.0.5 # via gitpython gitpython==3.1.7 # via transifex-client httplib2==0.18.1 # via -r dependencies/pip/requirements.in idna==2.10 # via requests -importlib-metadata==1.7.0 # via jsonschema, kombu, markdown, path jdcal==1.4.1 # via openpyxl jsonfield==3.1.0 # via -r dependencies/pip/requirements.in jsonschema==3.2.0 # via formpack @@ -71,8 +71,8 @@ numpy==1.19.0 # via pandas oauthlib==3.1.0 # via django-oauth-toolkit openpyxl==3.0.4 # via -r dependencies/pip/requirements.in pandas==1.0.5 # via -r dependencies/pip/requirements.in -path.py==12.4.0 # via -r dependencies/pip/requirements.in, formpack -path==13.1.0 # via path.py +path.py==12.5.0 # via formpack +path==15.0.0 # via path.py pillow==7.2.0 # via -r dependencies/pip/requirements.in, elaphe3 psycopg2-binary==2.8.5 # via -r dependencies/pip/requirements.in pymongo==3.10.1 # via -r dependencies/pip/requirements.in @@ -99,11 +99,11 @@ transifex-client==0.13.11 # via -r dependencies/pip/requirements.in unicodecsv==0.14.1 # via djangorestframework-csv, pyxform unittest2==1.1.0 # via pyxform urllib3==1.25.9 # via requests, transifex-client +uwsgi==2.0.19.1 # via -r dependencies/pip/prod.in vine==1.3.0 # via amqp, celery xlrd==1.2.0 # via -r dependencies/pip/requirements.in, pyxform xlsxwriter==1.2.9 # via formpack xlwt==1.3.0 # via -r dependencies/pip/requirements.in -zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt index 6aaaae70b..164d093a5 100644 --- a/dependencies/pip/requirements.txt +++ b/dependencies/pip/requirements.txt @@ -34,10 +34,11 @@ django-redis-sessions==0.6.1 # via -r dependencies/pip/requirements.in django-registration-redux==2.8 # via -r dependencies/pip/requirements.in django-render-block==0.7 # via django-templated-email django-reversion==3.0.1 # via -r dependencies/pip/requirements.in +django-storages==1.9.1 # via -r dependencies/pip/requirements.in django-taggit==1.3.0 # via -r dependencies/pip/requirements.in django-templated-email==2.3.0 # via -r dependencies/pip/requirements.in django-timezone-field==4.0 # via -r dependencies/pip/requirements.in, django-celery-beat -django==2.2.14 # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield +django==2.2.14 # via -r dependencies/pip/requirements.in, django-celery-beat, django-cors-headers, django-filter, django-guardian, django-oauth-toolkit, django-render-block, django-reversion, django-storages, django-taggit, django-timezone-field, djangorestframework, djangorestframework-guardian, jsonfield djangorestframework-csv==2.1.0 # via -r dependencies/pip/requirements.in djangorestframework-guardian==0.3.0 # via -r dependencies/pip/requirements.in djangorestframework-jsonp==1.0.2 # via -r dependencies/pip/requirements.in @@ -57,7 +58,6 @@ gitdb==4.0.5 # via gitpython gitpython==3.1.7 # via transifex-client httplib2==0.18.1 # via -r dependencies/pip/requirements.in idna==2.10 # via requests -importlib-metadata==1.7.0 # via jsonschema, kombu, markdown, path jdcal==1.4.1 # via openpyxl jsonfield==3.1.0 # via -r dependencies/pip/requirements.in jsonschema==3.2.0 # via formpack @@ -71,7 +71,7 @@ numpy==1.19.0 # via pandas oauthlib==3.1.0 # via django-oauth-toolkit openpyxl==3.0.4 # via -r dependencies/pip/requirements.in pandas==1.0.5 # via -r dependencies/pip/requirements.in -path.py==12.4.0 # via -r dependencies/pip/requirements.in, formpack +path.py==12.4.0 # via formpack path==13.1.0 # via path.py pillow==7.2.0 # via -r dependencies/pip/requirements.in, elaphe3 psycopg2-binary==2.8.5 # via -r dependencies/pip/requirements.in @@ -103,7 +103,6 @@ vine==1.3.0 # via amqp, celery xlrd==1.2.0 # via -r dependencies/pip/requirements.in, pyxform xlsxwriter==1.2.9 # via formpack xlwt==1.3.0 # via -r dependencies/pip/requirements.in -zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From 0a02cac7efa9bac226be9f390b5e298eb5d9fca8 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Wed, 19 Aug 2020 13:20:14 -0400 Subject: [PATCH 5/9] Fixed uwsgi binary path for wrong port warning --- docker/run_uwsgi_wrong_port_warning.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/run_uwsgi_wrong_port_warning.bash b/docker/run_uwsgi_wrong_port_warning.bash index ec8daca7d..2b53cebc1 100755 --- a/docker/run_uwsgi_wrong_port_warning.bash +++ b/docker/run_uwsgi_wrong_port_warning.bash @@ -8,7 +8,7 @@ source /etc/profile if [[ "${KOBOCAT_WEB_SERVER,,}" == "uwsgi" ]] then - exec /usr/local/bin/uwsgi --uid "${UWSGI_USER}" --socket :8000 --wsgi-file "${KOBOCAT_SRC_DIR}/docker/wrong_port_warning.wsgi" + exec $(command -v uwsgi) --uid "${UWSGI_USER}" --socket :8000 --wsgi-file "${KOBOCAT_SRC_DIR}/docker/wrong_port_warning.wsgi" else exec "${KOBOCAT_SRC_DIR}/docker/dev_wrong_port_warning.py" 8000 fi From 598f787ca5f051f87b2b474c1b680a0c72d988a9 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 22 Sep 2020 10:04:17 -0400 Subject: [PATCH 6/9] - Run `wait_for_*` scripts in `init.bash` at startup - Copy environment variables to $UWSGI_USER's bash rc --- Dockerfile | 9 ++++++--- docker/init.bash | 4 ++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2a6e1d035..51a7a3135 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,8 @@ ENV VIRTUAL_ENV=/opt/venv \ UWSGI_USER=kobo \ UWSGI_GROUP=kobo \ SERVICES_DIR=/srv/services \ - CELERY_PID_DIR=/var/run/celery + CELERY_PID_DIR=/var/run/celery \ + INIT_PATH=/srv/init # Install Dockerize ENV DOCKERIZE_VERSION v0.6.1 @@ -48,7 +49,8 @@ RUN mkdir -p ${NGINX_STATIC_DIR} && \ mkdir -p ${SERVICES_DIR}/celery && \ mkdir -p ${SERVICES_DIR}/celery_beat && \ mkdir -p ${KOBOCAT_LOGS_DIR}/ && \ - mkdir -p ${KOBOCAT_SRC_DIR}/emails + mkdir -p ${KOBOCAT_SRC_DIR}/emails && \ + mkdir -p ${INIT_PATH} # Install `apt` packages. RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - @@ -100,7 +102,8 @@ RUN pip-sync "${TMP_DIR}/pip_dependencies.txt" 1>/dev/null && \ # Using `/etc/profile.d/` as a repository for non-hard-coded environment variable overrides. RUN echo "export PATH=${PATH}" >> /etc/profile && \ - echo 'source /etc/profile' >> /root/.bashrc + echo 'source /etc/profile' >> /root/.bashrc && \ + echo 'source /etc/profile' >> /home/${UWSGI_USER}/.bashrc # Remove getty* services to avoid errors of absent tty at sv start-up RUN rm -rf /etc/runit/runsvdir/default/getty-tty* diff --git a/docker/init.bash b/docker/init.bash index 4f49a3158..89246b4ca 100755 --- a/docker/init.bash +++ b/docker/init.bash @@ -13,6 +13,10 @@ if [[ -z $DATABASE_URL ]]; then exit 1 fi +# Wait for databases to be up & running before going further +/bin/bash "${INIT_PATH}/wait_for_mongo.bash" +/bin/bash "${INIT_PATH}/wait_for_postgres.bash" + echo 'Running migrations...' python manage.py migrate --noinput From 504d3ccc9907fb98f5975e65daf0fe494f479397 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 8 Oct 2020 15:14:27 -0400 Subject: [PATCH 7/9] Delete old requirements --- requirements/README.md | 7 ---- requirements/base.pip | 82 ------------------------------------------ requirements/dev.pip | 22 ------------ requirements/s3.pip | 5 --- requirements/ses.pip | 2 -- 5 files changed, 118 deletions(-) delete mode 100644 requirements/README.md delete mode 100644 requirements/base.pip delete mode 100644 requirements/dev.pip delete mode 100644 requirements/s3.pip delete mode 100644 requirements/ses.pip diff --git a/requirements/README.md b/requirements/README.md deleted file mode 100644 index c346eb8d8..000000000 --- a/requirements/README.md +++ /dev/null @@ -1,7 +0,0 @@ -#REQUIREMENTS - -- ***base.pip*** - Contains requirements common to all environs and deploys. -- ***dev.pip*** - Requirements used for development and testing. - -- ***s3.pip*** - AWS S3 requirements used mainly in production. -- ***ses.pip*** - SES Mail backend requirements. diff --git a/requirements/base.pip b/requirements/base.pip deleted file mode 100644 index c59f47a28..000000000 --- a/requirements/base.pip +++ /dev/null @@ -1,82 +0,0 @@ -pytz==2018.9 - -Django>=1.8,<1.9 -dj-database-url==0.4.0 -django-guardian==1.4.1 -django-registration-redux==1.3 -django-templated-email==0.4.9 -gdata==2.0.18 -httplib2==0.9 -modilabs-python-utils==0.1.5 -Pillow==2.5.3 -poster==0.8.1 -psycopg2-binary==2.7.7 -pymongo==3.7.2 -lxml==3.4.0 -pyxform==0.13.1 -django-reversion==2.0.8 -xlrd==1.1.0 -xlwt==1.3.0 -openpyxl==2.0.5 -celery==4.2.1 -django-celery-beat==1.1.1 -amqp==2.4.0 -vine==1.2.0 -django-nose==1.4.2 -python-digest==1.7 -raven==6.1.0 --e git+https://github.com/dimagi/django-digest@0eb1c921329dd187c343b61acfbec4e98450136e#egg=django_digest - -# new export code relies on -pandas>=0.12.0 -elaphe==0.5.6 -requests==2.21.0 - -# formpack exports -git+https://github.com/kobotoolbox/formpack@kobocat-requirement -django-pure-pagination==0.3.0 -path.py==8.1.2 - -# sms support -dict2xml==1.3 - -# api support -djangorestframework==3.3.2 -djangorestframework-csv==1.4.1 -djangorestframework-jsonp==1.0.2 -djangorestframework-xml==1.3.0 - -# cors -django-cors-headers==0.13 -Markdown==2.5 -django-filter==0.7 - -# captcha -recaptcha-client==1.0.6 -unicodecsv==0.9.4 -dpath==1.2-70 - -# tagging -django-taggit==0.18.0 - -# oath2 support -django-oauth-toolkit==0.9.0 - -# spss -https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip#egg=savreaderwriter - -# JSON data type support -jsonfield<1.0 -django-db-readonly==0.3.2 - -transifex-client - -# Shell Plus -django-extensions==1.6.7 - -# redis-sessions -django-redis-sessions==0.6.1 -redis==3.2.0 - -# ssrf --e git+https://github.com/kobotoolbox/ssrf-protect@9eec6c4aa37700c6e7ca90540a9407bd20acddb0#egg=ssrf_protect diff --git a/requirements/dev.pip b/requirements/dev.pip deleted file mode 100644 index 99329a739..000000000 --- a/requirements/dev.pip +++ /dev/null @@ -1,22 +0,0 @@ -#clint -#django-kombu -#django-snippetscream -#django-statsd-mozilla -# fabric -# gunicorn -ipdb -ipython -shell_command -#statsd -#twill -Werkzeug -sqlparse -pytest -pytest-django -pytest-env -mongomock -mock - - -#python 3 -httmock diff --git a/requirements/s3.pip b/requirements/s3.pip deleted file mode 100644 index 277c5b31c..000000000 --- a/requirements/s3.pip +++ /dev/null @@ -1,5 +0,0 @@ -boto==2.48.0 - -# Necessary to use this fork until the resolution of -# https://github.com/jschneier/django-storages/issues/566 --e git+https://github.com/jnm/django-storages@s3boto_accurate_tell#egg=django_storages diff --git a/requirements/ses.pip b/requirements/ses.pip deleted file mode 100644 index 23c028d3f..000000000 --- a/requirements/ses.pip +++ /dev/null @@ -1,2 +0,0 @@ -boto==2.1.1 -django-ses==0.4.0 From a21dd37e9d222c65c64200f90657c1ca46e80868 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 8 Oct 2020 15:16:18 -0400 Subject: [PATCH 8/9] - Remove `USER` command from Dockerfile - Remove non-root user's password - Use uWSGI and Celery `--uid` and `--gid` option to start processes - Use gosu for other processes that don't have such option --- Dockerfile | 36 ++++-------------------- docker/init.bash | 8 +++--- docker/run_celery.bash | 4 ++- docker/run_celery_beat.bash | 4 ++- docker/run_uwsgi.bash | 28 +++++++++--------- docker/run_uwsgi_wrong_port_warning.bash | 5 ++-- docker/sync_static.bash | 2 +- 7 files changed, 33 insertions(+), 54 deletions(-) diff --git a/Dockerfile b/Dockerfile index 51a7a3135..e6afa3f86 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,5 @@ FROM nikolaik/python-nodejs:python3.8-nodejs10 -ARG DOCKER_USER_PASSWORD=kobo - -RUN if [ "$DOCKER_USER_PASSWORD" = "kobo" ]; then \ - echo "###################################################################" && \ - echo "# You are using default password: \`kobo\`. #" && \ - echo "# You should build your image with: #" && \ - echo "# \`docker build --build-arg DOCKER_USER_PASSWORD=\` #" && \ - echo "###################################################################" && \ - sleep 5; \ - fi; - # Declare environment variables ENV DEBIAN_FRONTEND=noninteractive ENV LANG=en_US.UTF-8 @@ -28,7 +17,7 @@ ENV VIRTUAL_ENV=/opt/venv \ TMP_DIR=/srv/tmp \ UWSGI_USER=kobo \ UWSGI_GROUP=kobo \ - SERVICES_DIR=/srv/services \ + SERVICES_DIR=/etc/service \ CELERY_PID_DIR=/var/run/celery \ INIT_PATH=/srv/init @@ -66,7 +55,7 @@ RUN apt-get -qq update && \ runit-init \ rsync \ vim \ - sudo \ + gosu \ cron && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -75,18 +64,8 @@ RUN apt-get -qq update && \ RUN echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen && \ locale-gen && dpkg-reconfigure locales -f noninteractive -# Create local user UWSGI_USER` with sudo privileges -RUN adduser --disabled-password --gecos '' "$UWSGI_USER" && \ - echo "${UWSGI_USER}:${DOCKER_USER_PASSWORD}" | chpasswd && \ - adduser "$UWSGI_USER" sudo - -# Add command to sudoers to avoid asking for `UWSGI_USER`'s password# -RUN echo "Cmnd_Alias SETUP_CRON = ${KOBOCAT_SRC_DIR}/docker/setup_cron.bash" >> /etc/sudoers && \ - echo "Cmnd_Alias SETUP_PYDEV_DEBUGGER = ${KOBOCAT_SRC_DIR}/docker/setup_pydev_debugger.bash" >> /etc/sudoers && \ - echo "Cmnd_Alias SYNC_STATIC = ${KOBOCAT_SRC_DIR}/docker/sync_static.bash" >> /etc/sudoers && \ - echo "$UWSGI_USER ALL=(ALL) NOPASSWD:SETENV: SETUP_CRON" >> /etc/sudoers && \ - echo "$UWSGI_USER ALL=(ALL) NOPASSWD:SETENV: SETUP_PYDEV_DEBUGGER" >> /etc/sudoers && \ - echo "$UWSGI_USER ALL=(ALL) NOPASSWD:SETENV: SYNC_STATIC" >> /etc/sudoers +# Create local user UWSGI_USER` +RUN adduser --disabled-password --gecos '' "$UWSGI_USER" # Copy KoBoCAT directory COPY . "${KOBOCAT_SRC_DIR}" @@ -117,18 +96,15 @@ RUN ln -s "${KOBOCAT_SRC_DIR}/docker/run_uwsgi_wrong_port_warning.bash" "${SERVI # Add/Restore `UWSGI_USER`'s permissions RUN chown -R ":${UWSGI_GROUP}" ${CELERY_PID_DIR} && \ chmod g+w ${CELERY_PID_DIR} && \ - chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${SERVICES_DIR} && \ chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${KOBOCAT_SRC_DIR}/emails/ && \ chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${KOBOCAT_LOGS_DIR} && \ chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${TMP_DIR} && \ - chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${VIRTUAL_ENV} + chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${VIRTUAL_ENV} && \ + chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${BACKUPS_DIR} WORKDIR "${KOBOCAT_SRC_DIR}" # TODO: Remove port 8000, say, at the start of 2021 (see kobotoolbox/kobo-docker#301 and wrong port warning above) EXPOSE 8001 8000 -# Run container with `UWSGI_USER` -USER $UWSGI_USER - CMD ["/bin/bash", "-c", "exec ${KOBOCAT_SRC_DIR}/docker/init.bash"] diff --git a/docker/init.bash b/docker/init.bash index 89246b4ca..0b635530c 100755 --- a/docker/init.bash +++ b/docker/init.bash @@ -18,12 +18,12 @@ fi /bin/bash "${INIT_PATH}/wait_for_postgres.bash" echo 'Running migrations...' -python manage.py migrate --noinput +gosu "${UWSGI_USER}" python manage.py migrate --noinput echo 'Setting up cron tasks...' -sudo -E ${KOBOCAT_SRC_DIR}/docker/setup_cron.bash -sudo -E ${KOBOCAT_SRC_DIR}/docker/setup_pydev_debugger.bash -sudo -E ${KOBOCAT_SRC_DIR}/docker/sync_static.bash +/bin/bash "${KOBOCAT_SRC_DIR}/docker/setup_cron.bash" +/bin/bash "${KOBOCAT_SRC_DIR}/docker/setup_pydev_debugger.bash" +/bin/bash "${KOBOCAT_SRC_DIR}/docker/sync_static.bash" echo 'Cleaning up Celery PIDs...' rm -rf ${CELERY_PID_DIR}/*.pid diff --git a/docker/run_celery.bash b/docker/run_celery.bash index 5af2966cb..f5af0deef 100755 --- a/docker/run_celery.bash +++ b/docker/run_celery.bash @@ -7,4 +7,6 @@ cd "${KOBOCAT_SRC_DIR}" exec celery worker -A onadata -Ofair --loglevel=info \ --hostname=kobocat_main_worker@%h \ --logfile=${KOBOCAT_LOGS_DIR}/celery.log \ - --pidfile=${CELERY_PID_DIR}/celery.pid + --pidfile=${CELERY_PID_DIR}/celery.pid \ + --uid=${UWSGI_USER} \ + --gid=${UWSGI_GROUP} diff --git a/docker/run_celery_beat.bash b/docker/run_celery_beat.bash index 19d522f16..faf270444 100755 --- a/docker/run_celery_beat.bash +++ b/docker/run_celery_beat.bash @@ -7,4 +7,6 @@ cd "${KOBOCAT_SRC_DIR}" exec celery beat -A onadata --loglevel=info \ --logfile=${KOBOCAT_LOGS_DIR}/celery_beat.log \ --pidfile=${CELERY_PID_DIR}/celery_beat.pid \ - --scheduler django_celery_beat.schedulers:DatabaseScheduler + --scheduler django_celery_beat.schedulers:DatabaseScheduler \ + --uid=${UWSGI_USER} \ + --gid=${UWSGI_GROUP} diff --git a/docker/run_uwsgi.bash b/docker/run_uwsgi.bash index 2f84c4592..463f4e810 100755 --- a/docker/run_uwsgi.bash +++ b/docker/run_uwsgi.bash @@ -4,7 +4,7 @@ source /etc/profile if [[ "$(stat -c '%U' ${KOBOCAT_LOGS_DIR})" != "${UWSGI_USER}" ]]; then echo 'Restoring ownership of Logs directory.' - chown -R "${UWSGI_USER}" "${KOBOCAT_LOGS_DIR}" + chown -R "${UWSGI_USER}":"${UWSGI_GROUP}" "${KOBOCAT_LOGS_DIR}" fi KOBOCAT_WEB_SERVER="${KOBOCAT_WEB_SERVER:-uWSGI}" @@ -13,17 +13,17 @@ if [[ "${KOBOCAT_WEB_SERVER,,}" == "uwsgi" ]]; then cd "${KOBOCAT_SRC_DIR}" DIFF=$(diff "${KOBOCAT_SRC_DIR}/dependencies/pip/prod.txt" "${TMP_DIR}/pip_dependencies.txt") if [[ -n "$DIFF" ]]; then - # Celery services need to be stopped to avoid raising errors during - # pip-sync + # Celery services need to be stopped to avoid raising errors + # during pip-sync echo "Stopping Celery services..." - sv stop "${SERVICES_DIR}/celery" - sv stop "${SERVICES_DIR}/celery_beat" + sv stop celery + sv stop celery_beat echo "Syncing pip dependencies..." pip-sync dependencies/pip/prod.txt 1>/dev/null cp "dependencies/pip/prod.txt" "${TMP_DIR}/pip_dependencies.txt" echo "Restarting Celery..." - sv start "${SERVICES_DIR}/celery" - sv start "${SERVICES_DIR}/celery_beat" + sv start celery + sv start celery_beat fi echo "Running \`KoBoCAT\` container with uWSGI application server." UWSGI_COMMAND="$(command -v uwsgi) --ini ${KOBOCAT_SRC_DIR}/docker/kobocat.ini" @@ -34,22 +34,22 @@ else # Celery services need to be stopped to avoid raising errors during # pip-sync echo "Stopping Celery services..." - sv stop "${SERVICES_DIR}/celery" - sv stop "${SERVICES_DIR}/celery_beat" + sv stop celery + sv stop celery_beat echo "Syncing pip dependencies..." - pip-sync dependencies/pip/dev.txt 1>/dev/null + gosu "${UWSGI_USER}" pip-sync dependencies/pip/dev.txt 1>/dev/null cp "dependencies/pip/dev.txt" "${TMP_DIR}/pip_dependencies.txt" echo "Restarting Celery..." - sv start "${SERVICES_DIR}/celery" - sv start "${SERVICES_DIR}/celery_beat" + sv start celery + sv start celery_beat fi if [[ -n "$RAVEN_DSN" ]]; then echo "Sentry detected. Installing \`raven\` pip dependency..." - pip install raven + gosu "${UWSGI_USER}" pip install raven fi echo "Running KoBoCAT container with \`runserver_plus\` debugging application server." - UWSGI_COMMAND="python manage.py runserver_plus 0:8001" + UWSGI_COMMAND="gosu $UWSGI_USER python manage.py runserver_plus 0:8001" fi exec ${UWSGI_COMMAND} diff --git a/docker/run_uwsgi_wrong_port_warning.bash b/docker/run_uwsgi_wrong_port_warning.bash index 2b53cebc1..61bae4375 100755 --- a/docker/run_uwsgi_wrong_port_warning.bash +++ b/docker/run_uwsgi_wrong_port_warning.bash @@ -5,10 +5,9 @@ source /etc/profile # Per kobotoolbox/kobo-docker#301, we have changed the uWSGI port to 8001. This # provides a helpful message to anyone still trying to use port 8000 - if [[ "${KOBOCAT_WEB_SERVER,,}" == "uwsgi" ]] then - exec $(command -v uwsgi) --uid "${UWSGI_USER}" --socket :8000 --wsgi-file "${KOBOCAT_SRC_DIR}/docker/wrong_port_warning.wsgi" + exec $(command -v uwsgi) --uid "${UWSGI_USER}" --gid "${UWSGI_GROUP}" --socket :8000 --wsgi-file "${KOBOCAT_SRC_DIR}/docker/wrong_port_warning.wsgi" else - exec "${KOBOCAT_SRC_DIR}/docker/dev_wrong_port_warning.py" 8000 + exec gosu "${UWSGI_USER}" "${KOBOCAT_SRC_DIR}/docker/dev_wrong_port_warning.py" 8000 fi diff --git a/docker/sync_static.bash b/docker/sync_static.bash index e1d963bd3..8fa164ec3 100755 --- a/docker/sync_static.bash +++ b/docker/sync_static.bash @@ -4,7 +4,7 @@ set -e mkdir -p "${KOBOCAT_SRC_DIR}/onadata/static" echo "Collecting static files..." -${VIRTUAL_ENV}/bin/python manage.py collectstatic -v 0 --noinput +gosu "${UWSGI_USER}" "${VIRTUAL_ENV}/bin/python" manage.py collectstatic -v 0 --noinput echo "Done" # `chown -R` becomes very slow once a fair amount of media has been collected, From b8bfc00d1b14f14b3cd4ff32fd32c9422ad9ebc3 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 9 Feb 2021 11:44:15 -0500 Subject: [PATCH 9/9] Create static folder as normal user --- docker/sync_static.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/sync_static.bash b/docker/sync_static.bash index 8fa164ec3..317887ab3 100755 --- a/docker/sync_static.bash +++ b/docker/sync_static.bash @@ -1,7 +1,7 @@ #!/bin/bash set -e -mkdir -p "${KOBOCAT_SRC_DIR}/onadata/static" +gosu "${UWSGI_USER}" mkdir -p "${KOBOCAT_SRC_DIR}/onadata/static" echo "Collecting static files..." gosu "${UWSGI_USER}" "${VIRTUAL_ENV}/bin/python" manage.py collectstatic -v 0 --noinput