build: add deployment container replicas and resource constraints (#1139

) * build: update backend image to use 1 uvicorn worker * build: add api deploy replicas, remove container_name * build: default to 1 local replica for easier debug * docs: update container name for docker logs cmd * fix: return of AuthUser during debug tests + linting
hotosm · Jan 25, 2024 · ecdf41a · ecdf41a
1 parent bb8b2ce
commit ecdf41a
Show file tree

Hide file tree

Showing 9 changed files with 48 additions and 38 deletions.
diff --git a/INSTALL.md b/INSTALL.md
@@ -196,7 +196,7 @@ https://{YOUR_DOMAIN}
 http://fmtm.localhost:7050
 ```
 
-> Note: If those link doesn't work, check the logs with `docker logs fmtm-api`.
+> Note: If those link doesn't work, check the logs with `docker compose logs api`.
 >
 > Note: Use `docker ps` to view all container names.
 

diff --git a/docker-compose.development.yml b/docker-compose.development.yml
@@ -17,6 +17,8 @@
 
 version: "3"
 
+name: fmtm-dev
+
 volumes:
   fmtm_frontend:
     name: fmtm-frontend-${GIT_BRANCH}
@@ -46,7 +48,6 @@ networks:
 services:
   proxy:
     image: "ghcr.io/hotosm/fmtm/proxy:${GIT_BRANCH}"
-    container_name: fmtm-${GIT_BRANCH}
     depends_on:
       api:
         condition: service_started
@@ -79,7 +80,6 @@ services:
 
   api:
     image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}"
-    container_name: fmtm-api-${GIT_BRANCH}
     volumes:
       - fmtm_logs:/opt/logs
       - fmtm_tiles:/opt/tiles
@@ -95,6 +95,15 @@ services:
     networks:
       - fmtm-net
     restart: "unless-stopped"
+    deploy:
+      replicas: ${API_REPLICAS:-2}
+      resources:
+        limits:
+          cpus: "0.9"
+          memory: 1500M
+        reservations:
+          cpus: "0.1"
+          memory: 100M
 
   ui:
     # This service simply builds the frontend to a volume
@@ -106,15 +115,13 @@ services:
       args:
         APP_VERSION: ${GIT_BRANCH}
         VITE_API_URL: https://${FMTM_API_DOMAIN:-api.${FMTM_DOMAIN}}
-    container_name: fmtm-ui-${GIT_BRANCH}
     volumes:
       - fmtm_frontend:/frontend
     network_mode: none
     restart: "on-failure:2"
 
   central:
     image: "ghcr.io/hotosm/fmtm/odkcentral:${ODK_CENTRAL_TAG:-v2023.5.0}"
-    container_name: fmtm-central-${GIT_BRANCH}
     depends_on:
       central-db:
         condition: service_healthy
@@ -147,15 +154,13 @@ services:
     # This service simply builds the frontend to a volume
     # accessible to the proxy, then shuts down
     image: "ghcr.io/hotosm/fmtm/odkcentral-ui:${ODK_CENTRAL_TAG:-v2023.5.0}"
-    container_name: fmtm-central-ui-${GIT_BRANCH}
     volumes:
       - central_frontend:/frontend
     network_mode: none
     restart: "on-failure:2"
 
   s3:
     image: "docker.io/minio/minio:${MINIO_TAG:-RELEASE.2024-01-01T16-36-33Z}"
-    container_name: fmtm-s3-${GIT_BRANCH}
     environment:
       MINIO_ROOT_USER: ${S3_ACCESS_KEY}
       MINIO_ROOT_PASSWORD: ${S3_SECRET_KEY}
@@ -177,7 +182,6 @@ services:
 
   fmtm-db:
     image: "postgis/postgis:${POSTGIS_TAG:-14-3.4-alpine}"
-    container_name: fmtm-db-${GIT_BRANCH}
     volumes:
       - fmtm_db_data:/var/lib/postgresql/data/
     environment:
@@ -198,7 +202,6 @@ services:
 
   central-db:
     image: "postgis/postgis:${POSTGIS_TAG:-14-3.4-alpine}"
-    container_name: fmtm-central-db-${GIT_BRANCH}
     volumes:
       - central_db_data:/var/lib/postgresql/data/
     environment:
@@ -219,7 +222,6 @@ services:
 
   migrations:
     image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}"
-    container_name: fmtm-migrations-${GIT_BRANCH}
     depends_on:
       fmtm-db:
         condition: service_healthy
@@ -234,7 +236,6 @@ services:
 
   certbot:
     image: "ghcr.io/hotosm/fmtm/proxy:certs-init-development"
-    container_name: fmtm-cert-renew-${GIT_BRANCH}
     volumes:
       - certs:/etc/letsencrypt
       - certbot_data:/var/www/certbot

diff --git a/docker-compose.main.yml b/docker-compose.main.yml
@@ -17,6 +17,8 @@
 
 version: "3"
 
+name: fmtm-main
+
 volumes:
   fmtm_frontend:
     name: fmtm-frontend-main
@@ -40,7 +42,6 @@ networks:
 services:
   proxy:
     image: "ghcr.io/hotosm/fmtm/proxy:main"
-    container_name: fmtm-main
     depends_on:
       api:
         condition: service_started
@@ -64,7 +65,6 @@ services:
 
   api:
     image: "ghcr.io/hotosm/fmtm/backend:main"
-    container_name: fmtm-api-main
     volumes:
       - fmtm_logs:/opt/logs
       - fmtm_tiles:/opt/tiles
@@ -78,26 +78,33 @@ services:
     networks:
       - fmtm-net
     restart: "unless-stopped"
+    deploy:
+      replicas: ${API_REPLICAS:-4}
+      resources:
+        limits:
+          cpus: "0.9"
+          memory: 1500M
+        reservations:
+          cpus: "0.1"
+          memory: 100M
 
   ui:
     # This service simply builds the frontend to a volume
     # accessible to the proxy, then shuts down
-    image: "ghcr.io/hotosm/fmtm/frontend:${GIT_BRANCH:-main}"
+    image: "ghcr.io/hotosm/fmtm/frontend:main"
     build:
       context: src/frontend
       dockerfile: prod.dockerfile
       args:
         APP_VERSION: main
         VITE_API_URL: https://${FMTM_API_DOMAIN:-api.${FMTM_DOMAIN}}
-    container_name: fmtm-ui-main
     volumes:
       - fmtm_frontend:/frontend
     network_mode: none
     restart: "on-failure:2"
 
   fmtm-db:
     image: "postgis/postgis:${POSTGIS_TAG:-14-3.4-alpine}"
-    container_name: fmtm-db-main
     volumes:
       - fmtm_db_data:/var/lib/postgresql/data/
     environment:
@@ -118,7 +125,6 @@ services:
 
   migrations:
     image: "ghcr.io/hotosm/fmtm/backend:main"
-    container_name: fmtm-migrations-main
     depends_on:
       fmtm-db:
         condition: service_healthy
@@ -131,7 +137,6 @@ services:
 
   backups:
     image: "ghcr.io/hotosm/fmtm/backend:main"
-    container_name: fmtm-backups-main
     depends_on:
       fmtm-db:
         condition: service_healthy
@@ -150,7 +155,6 @@ services:
 
   certbot:
     image: "ghcr.io/hotosm/fmtm/proxy:certs-init-main"
-    container_name: fmtm-cert-renew-main
     volumes:
       - certs:/etc/letsencrypt
       - certbot_data:/var/www/certbot

diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml
@@ -17,6 +17,8 @@
 
 version: "3"
 
+name: fmtm-stage
+
 volumes:
   fmtm_frontend:
     name: fmtm-frontend-${GIT_BRANCH}
@@ -56,7 +58,6 @@ services:
     extends:
       file: docker-compose.development.yml
       service: ui
-    image: "ghcr.io/hotosm/fmtm/frontend:${GIT_BRANCH:-staging}"
   central:
     extends:
       file: docker-compose.development.yml
@@ -83,7 +84,6 @@ services:
       service: migrations
   backups:
     image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}"
-    container_name: fmtm-backups-${GIT_BRANCH}
     depends_on:
       fmtm-db:
         condition: service_healthy

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -17,6 +17,8 @@
 
 version: "3"
 
+name: fmtm
+
 volumes:
   fmtm_data:
   fmtm_db_data:
@@ -38,7 +40,6 @@ services:
       target: debug
       args:
         NGINX_TAG: "${NGINX_TAG:-1.25.3}"
-    container_name: fmtm
     depends_on:
       api:
         condition: service_started
@@ -67,7 +68,6 @@ services:
       target: "${TARGET_OVERRIDE:-debug}"
       args:
         APP_VERSION: "${TAG_OVERRIDE:-debug}"
-    container_name: fmtm-api
     # Uncomment these to debug with a terminal debugger like pdb
     # Then `docker attach fmtm_api` to debug
     # stdin_open: true
@@ -94,18 +94,26 @@ services:
     env_file:
       - .env
     ports:
-      - "7052:8000"
-    #   - "5678:5678" # Debugger port
+      - "7052-7053:8000"
+    #   - "5678-5679:5678" # Debugger port
     networks:
       - fmtm-net
     restart: "unless-stopped"
+    deploy:
+      replicas: ${API_REPLICAS:-1}
+      resources:
+        limits:
+          cpus: "0.9"
+          memory: 1500M
+        reservations:
+          cpus: "0.1"
+          memory: 100M
 
   ui:
     image: "ghcr.io/hotosm/fmtm/frontend:debug"
     build:
       context: src/frontend
       dockerfile: debug.dockerfile
-    container_name: fmtm-ui
     depends_on:
       api:
         condition: service_started
@@ -130,7 +138,6 @@ services:
       context: odkcentral/api
       args:
         ODK_CENTRAL_TAG: ${ODK_CENTRAL_TAG:-v2023.5.0}
-    container_name: fmtm-central
     depends_on:
       central-db:
         condition: service_healthy
@@ -170,15 +177,13 @@ services:
       context: odkcentral/ui
       args:
         ODK_CENTRAL_TAG: ${ODK_CENTRAL_TAG:-v2023.5.0}
-    container_name: fmtm-central-ui
     volumes:
       - central_frontend:/frontend
     network_mode: none
     restart: "on-failure:2"
 
   s3:
     image: "docker.io/minio/minio:${MINIO_TAG:-RELEASE.2024-01-01T16-36-33Z}"
-    container_name: fmtm-s3
     environment:
       MINIO_ROOT_USER: ${S3_ACCESS_KEY:-fmtm}
       MINIO_ROOT_PASSWORD: ${S3_SECRET_KEY:-somelongpassword}
@@ -203,7 +208,6 @@ services:
 
   fmtm-db:
     image: "postgis/postgis:${POSTGIS_TAG:-14-3.4-alpine}"
-    container_name: fmtm-db
     volumes:
       - fmtm_db_data:/var/lib/postgresql/data/
     environment:
@@ -225,7 +229,6 @@ services:
   central-db:
     profiles: ["", "central"]
     image: "postgis/postgis:${POSTGIS_TAG:-14-3.4-alpine}"
-    container_name: fmtm-central-db
     volumes:
       - central_db_data:/var/lib/postgresql/data/
     environment:
@@ -246,7 +249,6 @@ services:
 
   migrations:
     image: "ghcr.io/hotosm/fmtm/backend:${TAG_OVERRIDE:-debug}"
-    container_name: fmtm-migrations
     depends_on:
       fmtm-db:
         condition: service_healthy

diff --git a/docs/dev/Backend.md b/docs/dev/Backend.md
@@ -39,7 +39,7 @@ URLs defined in the docker-compose file and your env file.
    `http://api.fmtm.localhost:7050/docs`
 
 > Note: If that link doesn't work, check the logs with
-> `docker log fmtm-api`.
+> `docker compose logs api`.
 > Note: the database host `fmtm-db` is automatically
 > resolved by docker compose to the database container IP.
 

diff --git a/src/backend/Dockerfile b/src/backend/Dockerfile
@@ -154,7 +154,7 @@ RUN pip install --user --upgrade --no-warn-script-location \
     && rm -r /opt/python
 CMD ["python", "-m", "debugpy", "--listen", "0.0.0.0:5678", \
     "-m", "uvicorn", "app.main:api", \
-    "--host", "0.0.0.0", "--port", "8000", "--workers", "4", \
+    "--host", "0.0.0.0", "--port", "8000", "--workers", "1", \
     "--reload", "--log-level", "critical", "--no-access-log"]
 
 
@@ -190,6 +190,6 @@ CMD ["sleep", "infinity"]
 FROM runtime as prod
 # Pre-compile packages to .pyc (init speed gains)
 RUN python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1)"
-# Note: 4 uvicorn workers as running with docker, change to 1 worker for Kubernetes
+# Note: 1 worker (process) per container, behind load balancer
 CMD ["uvicorn", "app.main:api", "--host", "0.0.0.0", "--port", "8000", \
-    "--workers", "4", "--log-level", "critical", "--no-access-log"]
+    "--workers", "1", "--log-level", "critical", "--no-access-log"]
diff --git a/src/backend/app/auth/osm.py b/src/backend/app/auth/osm.py
@@ -62,7 +62,7 @@ async def login_required(
 ) -> AuthUser:
     """Dependency to inject into endpoints requiring login."""
     if settings.DEBUG:
-        AuthUser(
+        return AuthUser(
             id=20386219,
             username="svcfmtm",
         )

diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py
@@ -214,11 +214,12 @@ async def read_project(project_id: int, db: Session = Depends(database.get_db)):
 
 @router.delete("/{project_id}")
 async def delete_project(
-    project: int = Depends(project_deps.get_project_by_id),
+    project: db_models.DbProject = Depends(project_deps.get_project_by_id),
     db: Session = Depends(database.get_db),
     user_data: AuthUser = Depends(login_required),
 ):
     """Delete a project from both ODK Central and the local database."""
+    log.info(f"User {user_data.username} attempting deletion of project {project.id}")
     # Odk crendentials
     odk_credentials = project_schemas.ODKCentral(
         odk_central_url=project.odk_central_url,
@@ -229,6 +230,8 @@ async def delete_project(
     await central_crud.delete_odk_project(project.odkid, odk_credentials)
     # Delete FMTM project
     await project_crud.delete_one_project(db, project)
+
+    log.info(f"Deletion of project {project.id} successful")
     return Response(status_code=HTTPStatus.NO_CONTENT)