Merge pull request #4948 from kobotoolbox/refactor-kobo-docker-env-vars

Remove `runsv` and KoboCAT uWSGI env variables

Dockerfile

@@ -29,8 +29,6 @@ ENV KPI_LOGS_DIR=/srv/logs \
     TMP_DIR=/srv/tmp \
     UWSGI_USER=kobo \
     UWSGI_GROUP=kobo \
-    SERVICES_DIR=/etc/service \
-    CELERY_PID_DIR=/var/run/celery \
     INIT_PATH=/srv/init
 
 ##########################################
@@ -40,14 +38,7 @@ ENV KPI_LOGS_DIR=/srv/logs \
 RUN mkdir -p "${NGINX_STATIC_DIR}" && \
     mkdir -p "${KPI_SRC_DIR}" && \
     mkdir -p "${KPI_NODE_PATH}" && \
-    mkdir -p "${TMP_DIR}" && \
-    mkdir -p ${CELERY_PID_DIR} && \
-    mkdir -p ${SERVICES_DIR}/uwsgi && \
-    mkdir -p ${SERVICES_DIR}/celery_kpi && \
-    mkdir -p ${SERVICES_DIR}/celery_kobocat && \
-    mkdir -p ${SERVICES_DIR}/celery_low_priority && \
-    mkdir -p ${SERVICES_DIR}/celery_beat && \
-    mkdir -p "${INIT_PATH}"
+    mkdir -p "${TMP_DIR}"
 
 ##########################################
 # Install `apt` packages.                #
@@ -78,7 +69,6 @@ RUN apt-get -qq update && \
         postgresql-client \
         procps \
         rsync \
-        runit-init \
         vim-tiny \
         wait-for-it && \
     apt-get clean && \
@@ -163,29 +153,15 @@ RUN echo "export PATH=${PATH}" >> /etc/profile && \
     echo 'source /etc/profile' >> /root/.bashrc && \
     echo 'source /etc/profile' >> /home/${UWSGI_USER}/.bashrc
 
-
-# Remove getty* services to avoid errors of absent tty at sv start-up
-RUN rm -rf /etc/runit/runsvdir/default/getty-tty*
-
-# Create symlinks for runsv services
-RUN ln -s "${KPI_SRC_DIR}/docker/run_uwsgi.bash" "${SERVICES_DIR}/uwsgi/run" && \
-    ln -s "${KPI_SRC_DIR}/docker/run_celery_kpi.bash" "${SERVICES_DIR}/celery_kpi/run" && \
-    ln -s "${KPI_SRC_DIR}/docker/run_celery_kobocat.bash" "${SERVICES_DIR}/celery_kobocat/run" && \
-    ln -s "${KPI_SRC_DIR}/docker/run_celery_low_priority.bash" "${SERVICES_DIR}/celery_low_priority/run" && \
-    ln -s "${KPI_SRC_DIR}/docker/run_celery_beat.bash" "${SERVICES_DIR}/celery_beat/run"
-
-
 # Add/Restore `UWSGI_USER`'s permissions
 # chown of `${TMP_DIR}/.npm` is a hack needed for kobo-install-based staging deployments;
 # see internal discussion at https://chat.kobotoolbox.org/#narrow/stream/4-Kobo-Dev/topic/Unu.2C.20du.2C.20tri.2C.20kvar.20deployments/near/322075
-RUN chown -R ":${UWSGI_GROUP}" ${CELERY_PID_DIR} && \
-    chmod g+w ${CELERY_PID_DIR} && \
-    chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${KPI_SRC_DIR}/emails/ && \
+RUN chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${KPI_SRC_DIR}/emails/ && \
     chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${KPI_LOGS_DIR} && \
     chown -R "${UWSGI_USER}:${UWSGI_GROUP}" ${TMP_DIR} && \
     chown -R root:root "${TMP_DIR}/.npm"
 
 
 EXPOSE 8000
 
-CMD ["/bin/bash", "-c", "exec ${KPI_SRC_DIR}/docker/init.bash"]
+CMD ["/bin/bash", "docker/entrypoint.sh"]

README.md

@@ -6,9 +6,17 @@
 
 We're open for [contributions](./CONTRIBUTING.md)!
 
+## Important notice when upgrading from any release older than [`2.024.19`](https://github.com/kobotoolbox/kpi/releases/tag/2.024.19)
+
+Prior to release [`2.024.19`](https://github.com/kobotoolbox/kpi/releases/tag/2.024.19), this project (KPI) and [KoboCAT](https://github.com/kobotoolbox/kobocat) were two separated projects.
+KoboCAT is now part of KPI code base and its repository has been archived.
+
+KoboCAT deprecation notices will be maintained in this repository.
+[More details here](./kobo/apps/openrosa/README.md)
+
 ## Important notice when upgrading from any release older than [`2.020.18`](https://github.com/kobotoolbox/kpi/releases/tag/2.020.18)
 
-Prior to release [`2.020.18`](https://github.com/kobotoolbox/kpi/releases/tag/2.020.18), this project (KPI) and [KoBoCAT](https://github.com/kobotoolbox/kobocat) both shared a common Postgres database. They now each have their own. **If you are upgrading an existing single-database installation, you must follow [these instructions](https://community.kobotoolbox.org/t/upgrading-to-separate-databases-for-kpi-and-kobocat/7202)** to migrate the KPI tables to a new database and adjust your configuration appropriately.
+Prior to release [`2.020.18`](https://github.com/kobotoolbox/kpi/releases/tag/2.020.18), this project (KPI) and [KoboCAT](https://github.com/kobotoolbox/kobocat) both shared a common Postgres database. They now each have their own. **If you are upgrading an existing single-database installation, you must follow [these instructions](https://community.kobotoolbox.org/t/upgrading-to-separate-databases-for-kpi-and-kobocat/7202)** to migrate the KPI tables to a new database and adjust your configuration appropriately.
 
 If you do not want to upgrade at this time, please use the [`shared-database-obsolete`](https://github.com/kobotoolbox/kpi/tree/shared-database-obsolete) branch instead.
 
@@ -35,7 +43,7 @@ syntax, see the documentation at the top of
 
 ## Admin reports
 
-There are several types of data reports available to superusers. 
+There are several types of data reports available to superusers.
 * Full list of users including their details provided during signup, number of deployed projects (XForm count), number of submissions, date joined, and last login: `<kpi base url>/superuser_stats/user_report/`. File being created is a CSV, so don't download immediately to wait for server to be finished writing to the file (it will download even if incomplete).
 * Monthly aggregate figures for number of forms, deployed projects, and submissions (from kobocat): `<kc server domain>/<superuser username>/superuser_stats/`
 

docker/init.bash → docker/entrypoint.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
 set -e
-
 source /etc/profile
 
 echo 'KPI initializing…'
@@ -14,8 +13,8 @@ if [[ -z $DATABASE_URL ]]; then
 fi
 
 # Handle Python dependencies BEFORE attempting any `manage.py` commands
-KPI_WEB_SERVER="${KPI_WEB_SERVER:-uWSGI}"
-if [[ "${KPI_WEB_SERVER,,}" == 'uwsgi' ]]; then
+WSGI="${WSGI:-uWSGI}"
+if [[ "${WSGI}" == 'uWSGI' ]]; then
     # `diff` returns exit code 1 if it finds a difference between the files
     if ! diff -q "${KPI_SRC_DIR}/dependencies/pip/requirements.txt" "${TMP_DIR}/pip_dependencies.txt"
     then
@@ -37,7 +36,7 @@ fi
 /bin/bash "${INIT_PATH}/wait_for_postgres.bash"
 
 echo 'Running migrations…'
-gosu "${UWSGI_USER}" python manage.py migrate --noinput
+gosu "${UWSGI_USER}" scripts/migrate.sh
 
 echo 'Creating superuser…'
 gosu "${UWSGI_USER}" python manage.py create_kobo_superuser
@@ -77,16 +76,10 @@ if [[ ! -d "${KPI_SRC_DIR}/locale" ]] || [[ -z "$(ls -A ${KPI_SRC_DIR}/locale)"
     python manage.py compilemessages
 fi
 
-rm -rf /etc/profile.d/pydev_debugger.bash.sh
-if [[ -d /srv/pydev_orig && -n "${KPI_PATH_FROM_ECLIPSE_TO_PYTHON_PAIRS}" ]]; then
-    echo 'Enabling PyDev remote debugging.'
-    "${KPI_SRC_DIR}/docker/setup_pydev.bash"
-fi
-
 echo 'Cleaning up Celery PIDs…'
 rm -rf /tmp/celery*.pid
 
-echo 'Restore permissions on Celery logs folder'
+echo 'Restore permissions on logs folder'
 chown -R "${UWSGI_USER}:${UWSGI_GROUP}" "${KPI_LOGS_DIR}"
 
 # This can take a while when starting a container with lots of media files.
@@ -96,4 +89,12 @@ chown -R "${UWSGI_USER}:${UWSGI_GROUP}" "${KPI_MEDIA_DIR}"
 
 echo 'KPI initialization completed.'
 
-exec /usr/bin/runsvdir "${SERVICES_DIR}"
+cd "${KPI_SRC_DIR}"
+
+if [[ "${WSGI}" == 'uWSGI' ]]; then
+    echo "Running \`kpi\` container with uWSGI application server."
+    $(command -v uwsgi) --ini ${KPI_SRC_DIR}/docker/uwsgi.ini
+else
+    echo "Running \`kpi\` container with \`runserver_plus\` debugging application server."
+    python manage.py runserver_plus 0:8000
+fi

docker/run_celery_beat.bash → docker/entrypoint_celery_beat.bash

@@ -4,6 +4,7 @@ source /etc/profile
 
 # Run the main Celery worker (will not process `sync_kobocat_xforms` jobs).
 cd "${KPI_SRC_DIR}"
+
 exec celery -A kobo beat --loglevel=info \
     --logfile=${KPI_LOGS_DIR}/celery_beat.log \
     --pidfile=/tmp/celery_beat.pid \

docker/run_celery_kobocat.bash → docker/entrypoint_celery_kobocat_worker.bash

@@ -10,9 +10,9 @@ AUTOSCALE_MIN="${CELERY_AUTOSCALE_MIN:-2}"
 AUTOSCALE_MAX="${CELERY_AUTOSCALE_MAX:-6}"
 
 exec celery -A kobo worker --loglevel=info \
-    --hostname=kobocat_main_worker@%h \
-    --logfile=${KPI_LOGS_DIR}/celery_kobocat.log \
-    --pidfile=/tmp/celery_kobocat.pid \
+    --hostname=kobocat_worker@%h \
+    --logfile=${KPI_LOGS_DIR}/celery_kobocat_worker.log \
+    --pidfile=/tmp/celery_kobocat_worker.pid \
     --queues=kobocat_queue \
     --exclude-queues=kpi_low_priority_queue,kpi_queue \
     --uid=${UWSGI_USER} \

docker/run_celery_low_priority.bash → docker/entrypoint_celery_kpi_low_priority_worker.bash

@@ -10,9 +10,9 @@ AUTOSCALE_MIN="${CELERY_AUTOSCALE_MIN:-2}"
 AUTOSCALE_MAX="${CELERY_AUTOSCALE_MAX:-6}"
 
 exec celery -A kobo worker --loglevel=info \
-    --hostname=kpi_main_worker@%h \
-    --logfile=${KPI_LOGS_DIR}/celery_low_priority.log \
-    --pidfile=/tmp/celery_low_priority.pid \
+    --hostname=kpi_low_priority_worker@%h \
+    --logfile=${KPI_LOGS_DIR}/celery_kpi_low_priority_worker.log \
+    --pidfile=/tmp/celery_kpi_low_priority_worker.pid \
     --queues=kpi_low_priority_queue \
     --exclude-queues=kpi_queue,kobocat_queue \
     --uid=${UWSGI_USER} \

docker/run_celery_kpi.bash → docker/entrypoint_celery_kpi_worker.bash

@@ -10,9 +10,9 @@ AUTOSCALE_MIN="${CELERY_AUTOSCALE_MIN:-2}"
 AUTOSCALE_MAX="${CELERY_AUTOSCALE_MAX:-6}"
 
 exec celery -A kobo worker --loglevel=info \
-    --hostname=kpi_main_worker@%h \
-    --logfile=${KPI_LOGS_DIR}/celery_kpi.log \
-    --pidfile=/tmp/celery_kpi.pid \
+    --hostname=kpi_worker@%h \
+    --logfile=${KPI_LOGS_DIR}/celery_kpi_worker.log \
+    --pidfile=/tmp/celery_kpi_worker.pid \
     --queues=kpi_queue \
     --exclude-queues=kpi_low_priority_queue,kobocat_queue \
     --uid=${UWSGI_USER} \

docker/uwsgi.ini

@@ -13,23 +13,23 @@ mount              = $(KPI_PREFIX)=$(KPI_SRC_DIR)/kobo/wsgi.py
 
 # process related settings
 master              = true
-harakiri            = $(KPI_UWSGI_HARAKIRI)
-worker-reload-mercy = $(KPI_UWSGI_WORKER_RELOAD_MERCY)
+harakiri            = $(UWSGI_HARAKIRI)
+worker-reload-mercy = $(UWSGI_WORKER_RELOAD_MERCY)
 
 # monitoring (use with `uwsgitop :1717`, for example)
 stats = :1717
 memory-report = true
 
 # Overrideable default of 2 uWSGI processes.
-if-env = KPI_UWSGI_WORKERS_COUNT
+if-env = UWSGI_WORKERS_COUNT
 workers = %(_)
 endif =
-if-not-env = KPI_UWSGI_WORKERS_COUNT
+if-not-env = UWSGI_WORKERS_COUNT
 workers = 2
 endif =
 
 # activate cheaper mode
-if-env = KPI_UWSGI_CHEAPER_WORKERS_COUNT
+if-env = UWSGI_CHEAPER_WORKERS_COUNT
 cheaper-algo = busyness
 cheaper = %(_)
 cheaper-initial = %(_)
@@ -41,23 +41,23 @@ cheaper-busyness-multiplier = 20
 endif =
 
 # stop spawning new workers if total memory consumption grows too large
-if-env = KPI_UWSGI_CHEAPER_RSS_LIMIT_SOFT
+if-env = UWSGI_CHEAPER_RSS_LIMIT_SOFT
 cheaper-rss-limit-soft = %(_)
 endif =
-if-not-env = KPI_UWSGI_CHEAPER_RSS_LIMIT_SOFT
+if-not-env = UWSGI_CHEAPER_RSS_LIMIT_SOFT
 cheaper-rss-limit-soft = %(2 * 1024 * 1024 * 1024)
 endif =
 
 # respawn processes after serving KPI_UWSGI_MAX_REQUESTS requests (default 5000)
-if-env = KPI_UWSGI_MAX_REQUESTS
+if-env = UWSGI_MAX_REQUESTS
 max-requests = %(_)
 endif =
 
 # respawn workers when their memory consumption grows too large
-if-env = KPI_UWSGI_RELOAD_ON_RSS_MB
+if-env = UWSGI_RELOAD_ON_RSS_MB
 reload-on-rss = %(_)
 endif =
-if-not-env = KPI_UWSGI_RELOAD_ON_RSS_MB
+if-not-env = UWSGI_RELOAD_ON_RSS_MB
 reload-on-rss = 512
 endif =
 

hub/admin/password_validation.py

@@ -3,7 +3,7 @@
 from django.db import transaction
 from django.utils.html import format_html
 
-from kpi.deployment_backends.kc_access.shadow_models import KobocatUserProfile
+from kobo.apps.openrosa.apps.main.models import UserProfile
 from .filters import PasswordValidationAdvancedSearchFilter
 from .mixins import AdvancedSearchMixin
 from ..models import ExtraUserDetail
@@ -67,12 +67,17 @@ def get_queryset(self, request):
 
     @admin.display(description='Validated')
     def get_validated_password(self, obj):
-        value = True
+        value = False
         try:
             value = obj.extra_details.validated_password
         except obj.extra_details.RelatedObjectDoesNotExist:
             pass
 
+        try:
+            value = value and obj.profile.validated_password
+        except obj.profile.RelatedObjectDoesNotExist:
+            pass
+
         return format_html(
             '<img src="/static/admin/img/icon-{}.svg" alt="{}">',
             'yes' if value else 'no',
@@ -88,8 +93,14 @@ def invalidate_passwords(self, request, queryset, **kwargs):
             ExtraUserDetail.objects.filter(user_id__in=user_ids).update(
                 validated_password=False
             )
-            KobocatUserProfile.objects.filter(user_id__in=user_ids).update(
-                validated_password=False
+            UserProfile.objects.bulk_create(
+                [
+                    UserProfile(user_id=user_id, validated_password=False)
+                    for user_id in user_ids
+                ],
+                update_conflicts=True,
+                unique_fields=['user_id'],
+                update_fields=['validated_password'],
             )
 
         self.message_user(
@@ -107,8 +118,14 @@ def validate_passwords(self, request, queryset, **kwargs):
             ExtraUserDetail.objects.filter(user_id__in=user_ids).update(
                 validated_password=True
             )
-            KobocatUserProfile.objects.filter(user_id__in=user_ids).update(
-                validated_password=True
+            UserProfile.objects.bulk_create(
+                [
+                    UserProfile(user_id=user_id, validated_password=True)
+                    for user_id in user_ids
+                ],
+                update_conflicts=True,
+                unique_fields=['user_id'],
+                update_fields=['validated_password'],
             )
 
         self.message_user(

jsapp/js/components/formGallery/formGallery.component.tsx

@@ -167,7 +167,7 @@ export default function FormGallery(props: FormGalleryProps) {
               target='_blank'
             >
               <img
-                src={attachment.download_url}
+                src={attachment.download_small_url}
                 alt={attachment.filename}
                 width='150'
                 loading='lazy'