diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml new file mode 100644 index 00000000..5d7138e9 --- /dev/null +++ b/.github/workflows/build_and_deploy.yml @@ -0,0 +1,95 @@ +name: Build and Deploy Drone Tasking Manager + +on: + push: + branches: + - main + workflow_dispatch: + +permissions: + id-token: write + contents: read + +jobs: + build: + name: Build Docker image + runs-on: ubuntu-latest + environment: + name: ${{ github.ref_name }} + steps: + - name: Clone repository + uses: actions/checkout@v3 + with: + ref: ${{ github.ref }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: naxa + password: ${{ secrets.DOCKERHUB_PAT }} + + - name: Build and push backend image + uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: naxa/dronetm:backend + file: ./src/backend/Dockerfile + + - name: Write Environment Variables for Frontend + run: | + echo ${{ vars.FRONTEND_ENV_VARS }} > .env + + - name: Build and push frontend image + uses: docker/build-push-action@v6 + with: + context: . + push: true + target: live + tags: naxa/dronetm:frontend + file: ./src/frontend/Dockerfile + + deploy_to_vm: + name: Deploy to VM + needs: + - build + runs-on: ubuntu-latest + environment: + name: ${{ github.ref_name }} + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: naxa + password: ${{ secrets.DOCKERHUB_PAT }} + + - name: Setup SSH Key + uses: webfactory/ssh-agent@v0.8.0 + with: + ssh-private-key: "${{ secrets.SSH_PRIVATE_KEY }}" + + - name: Add host keys to known_hosts + run: | + ssh-keyscan "${{ secrets.SSH_HOST }}" >> ~/.ssh/known_hosts + + - name: create env file + run: | + echo '${{ secrets.BACKEND_ENV_VARS }}' > .env + + - name: Deploy to VM + run: | + docker compose --file docker-compose.vm.yml --env-file .env pull + docker compose --file docker-compose.vm.yml --env-file .env up \ + --detach --remove-orphans --force-recreate + env: + DOCKER_HOST: "ssh://${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }}" + diff --git a/.github/workflows/build_and_deploy_DTM_backend.yml b/.github/workflows/build_and_deploy_DTM_backend.yml deleted file mode 100644 index f8d9a50b..00000000 --- a/.github/workflows/build_and_deploy_DTM_backend.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Build and Deploy Drone Tasking Manager Backend - -on: - push: - branches: - - main - paths: - - src/backend/** - workflow_dispatch: - -permissions: - id-token: write - contents: read - -env: - AWS_REGION: ap-south-1 - ECR_REGISTRY: 685797548389.dkr.ecr.ap-south-1.amazonaws.com - ECR_REPOSITORY: dtmweb - -jobs: - build: - name: Build Docker image - runs-on: ubuntu-latest - environment: - name: ${{ github.ref_name }} - steps: - - name: Clone repository - uses: actions/checkout@v3 - with: - ref: ${{ github.ref }} - - - name: Setup AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - audience: sts.amazonaws.com - aws-region: ${{ env.AWS_REGION }} - role-session-name: GH-Actions-${{ github.run_id }}-${{ github.run_attempt }} - role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Build and Push Docker Image - id: build-image - run: | - docker build -f ./src/backend/Dockerfile -t ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ github.ref_name }} . - docker push ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ github.ref_name }} - - - name: Image Digest - run: echo ImageDigest:${{ steps.build-image.outputs.imageDigest }} - - deploy_to_ecs: - name: Deploy to ECS - needs: - - build - runs-on: ubuntu-latest - environment: - name: ${{ github.ref_name }} - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Setup AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - audience: sts.amazonaws.com - aws-region: ${{ env.AWS_REGION }} - role-session-name: GH-Actions-${{ github.run_id }}-${{ github.run_attempt }} - role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} - - - name: Get Web App task definition - id: get-web-app-task-definition - shell: bash - run: | - TASK_DEFINITION_ARN=$(aws ecs describe-task-definition --task-definition fastapi-tdf --region ${{ env.AWS_REGION }} --query 'taskDefinition.taskDefinitionArn') - echo "TASK_DEFINITION_ARN=${TASK_DEFINITION_ARN}" >> $GITHUB_OUTPUT - - - name: ECS Service - uses: scribd/amazon-ecs-service-action@v1 - with: - force-new-deployment: true - spec: | - { - "taskDefinition": ${{ steps.get-web-app-task-definition.outputs.TASK_DEFINITION_ARN }}, - "cluster": "${{ secrets.ECS_CLUSTER_NAME }}", - "serviceName": "${{ secrets.ECS_SERVICE_NAME }}" - } - wait-until-deployment-complete: true diff --git a/.github/workflows/build_and_deploy_DTM_frontend.yml b/.github/workflows/build_and_deploy_DTM_frontend.yml deleted file mode 100644 index 0028ebf6..00000000 --- a/.github/workflows/build_and_deploy_DTM_frontend.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Build and Deploy Drone Tasking Manager Frontend - -on: - push: - branches: - - main - paths: - - src/frontend/** - workflow_dispatch: - -permissions: - id-token: write - contents: read - -env: - AWS_REGION: ap-south-1 - S3_BUCKET: dronetm.naxa.com.np - -jobs: - build: - name: Build JavaScript assets - runs-on: ubuntu-latest - environment: - name: ${{ github.ref_name }} - steps: - - name: Clone repository - uses: actions/checkout@v3 - with: - ref: ${{ github.ref }} - - - name: Use Node.js 19.x - uses: actions/setup-node@v1 - with: - node-version: 19.x - - - name: Install yarn - working-directory: ./src/frontend/ - run: npm install -g yarn - - - name: Cache node_modules - uses: actions/cache@v2 - with: - path: ./src/frontend/node_modules - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-build-${{ env.cache-name }}- - ${{ runner.os }}-build- - ${{ runner.os }}- - - - name: Write Environment Variables - id: write_env - working-directory: ./src/frontend/ - run: | - echo ${{ vars.FRONTEND_ENV_VARS }} > .env - - - name: Install dependencies - working-directory: ./src/frontend/ - run: yarn - - - name: Generate build - working-directory: ./src/frontend/ - run: | - yarn build - - - name: Setup AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - audience: sts.amazonaws.com - aws-region: ${{ env.AWS_REGION }} - role-session-name: GH-Actions-${{ github.run_id }}-${{ github.run_attempt }} - role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} - - - name: Copy Static Files to S3 - working-directory: ./src/frontend/ - run: | - pwd - ls -alh - aws s3 cp --recursive ./dist s3://${{ env.S3_BUCKET }} diff --git a/.github/workflows/migrations.yml b/.github/workflows/migrations.yml deleted file mode 100644 index 14317a2e..00000000 --- a/.github/workflows/migrations.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Database Migrations - -on: - workflow_dispatch: - -permissions: - id-token: write - contents: read - -env: - AWS_REGION: ap-south-1 - ECR_REGISTRY: 685797548389.dkr.ecr.ap-south-1.amazonaws.com - ECR_REPOSITORY: dtmweb - -jobs: - migration: - name: Do migration on RDS - runs-on: ubuntu-latest - environment: - name: ${{ github.ref_name }} - steps: - - name: Clone repository - uses: actions/checkout@v3 - with: - ref: ${{ github.ref }} - - - name: Setup AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - audience: sts.amazonaws.com - aws-region: ${{ env.AWS_REGION }} - role-session-name: GH-Actions-${{ github.run_id }}-${{ github.run_attempt }} - role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} - - - name: Run migration command - id: run-migration - shell: bash - run: | - TASK_ID=`aws ecs list-tasks --cluster ${{ secrets.ECS_CLUSTER_NAME }} --service ${{ secrets.ECS_SERVICE_NAME }} | jq -r .taskArns[0]` - aws ecs execute-command --cluster ${{ secrets.ECS_CLUSTER_NAME }} --task $TASK_ID --region ${{env.AWS_REGION}} --interactive --command "/bin/bash -c 'pdm run alembic upgrade head'" diff --git a/.github/workflows/test_frontend.yml b/.github/workflows/test_frontend.yml deleted file mode 100644 index 635faeab..00000000 --- a/.github/workflows/test_frontend.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Test Frontend Static Build - -on: - pull_request: - branches: - - main - - develop - - staging - paths: - - src/frontend/** - workflow_dispatch: - -permissions: - id-token: write - contents: read - -jobs: - build: - name: Build JavaScript assets - runs-on: ubuntu-latest - environment: - name: ${{ github.ref_name }} - steps: - - name: Clone repository - uses: actions/checkout@v3 - with: - ref: ${{ github.ref }} - - - name: Use Node.js 19.x - uses: actions/setup-node@v1 - with: - node-version: 19.x - - - name: Install yarn - working-directory: ./src/frontend/ - run: npm install -g yarn - - - name: Cache node_modules - uses: actions/cache@v2 - with: - path: ./src/frontend/node_modules - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-build-${{ env.cache-name }}- - ${{ runner.os }}-build- - ${{ runner.os }}- - - - name: Write Environment Variables - id: write_env - working-directory: ./src/frontend/ - run: | - echo ${{ vars.FRONTEND_ENV_VARS }} > .env - - - name: Install dependencies - working-directory: ./src/frontend/ - run: yarn - - - name: Generate build - working-directory: ./src/frontend/ - run: | - yarn build diff --git a/docker-compose.vm.yml b/docker-compose.vm.yml new file mode 100644 index 00000000..92eec82e --- /dev/null +++ b/docker-compose.vm.yml @@ -0,0 +1,82 @@ +version: "3" + +services: + backend: + image: naxa/dronetm:${BACKEND_TARGET_OVERRIDE:-backend} + restart: always + depends_on: + - db + - minio + ports: + - ${BACKEND_WEB_APP_PORT:-8000}:8000 + env_file: .env + networks: + - dtm-network + + frontend: + image: naxa/dronetm:${FRONTEND_TARGET_OVERRIDE:-frontend} + depends_on: + - minio + env_file: .env + networks: + - dtm-network + + db: + image: postgis/postgis:14-3.4-alpine + restart: always + volumes: + - ${PROJECT_DIR:-.}/DockerData/dtm_db_data:/var/lib/postgresql/data/ + env_file: .env + networks: + - dtm-network + healthcheck: + test: pg_isready -U ${POSTGRES_USER:-dtm} -d ${POSTGRES_DB:-dtm_db} + start_period: 5s + interval: 10s + timeout: 5s + retries: 3 + + minio: + image: "docker.io/minio/minio:${MINIO_TAG:-RELEASE.2023-10-25T06-33-25Z}" + restart: always + command: server /export --console-address 0.0.0.0:9090 --address 0.0.0.0:9000 + volumes: + - ${PROJECT_DIR:-.}/DockerData/minio_data:/export + environment: + MINIO_ROOT_USER: ${S3_ACCESS_KEY:-dtm_user} + MINIO_ROOT_PASSWORD: ${S3_SECRET_KEY:-somelongpassword} + MINIO_CONSOLE_ADDRESS: ":9090" + ports: + - 9000:9000 + - 9090:9090 + networks: + - dtm-network + + createbuckets: + image: "docker.io/minio/minio:${MINIO_TAG:-RELEASE.2023-10-25T06-33-25Z}" + entrypoint: > + /bin/sh -c " mc config host add minio http://minio:9000 \$S3_ACCESS_KEY \$S3_SECRET_KEY; mc mb minio/\$S3_BUCKET_NAME; mc anonymous set download minio/\$S3_BUCKET_NAME/publicuploads/; exit 0; " + env_file: + - .env + depends_on: + - minio + networks: + - dtm-network + + migrations: + image: naxa/dronetm:${BACKEND_TARGET_OVERRIDE:-backend} + volumes: + - ${PROJECT_DIR:-.}/src/backend:/project/src/backend + depends_on: + - backend + - db + env_file: + - .env + networks: + - dtm-network + entrypoint: ["pdm", "run", "alembic", "upgrade", "head"] + restart: "no" + +networks: + dtm-network: + name: dtm-network diff --git a/docker-compose.yml b/docker-compose.yml index 0335796a..da50c718 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ services: ports: - ${BACKEND_WEB_APP_PORT:-8000}:8000 volumes: - - ./src/backend:/project/src/backend + - ${PROJECT_DIR:-.}/src/backend:/project/src/backend env_file: .env networks: - dtm-network @@ -21,7 +21,7 @@ services: build: context: . dockerfile: src/frontend/Dockerfile - target: ${DOCKER_TARGET:-development} + target: ${FRONTEND_TARGET_OVERRIDE:-development} ports: - ${FRONTEND_WEB_APP_PORT:-3040}:3040 depends_on: @@ -30,19 +30,17 @@ services: networks: - dtm-network volumes: - - ./src/frontend:/app + - ${PROJECT_DIR:-.}/src/frontend:/app - /var/run/docker.sock:/var/run/docker.sock - - ./src/frontend/docker-entrypoint.sh:/docker-entrypoint.sh db: image: postgis/postgis:14-3.4-alpine restart: always volumes: - - ./DockerData/dtm_db_data:/var/lib/postgresql/data/ + - ${PROJECT_DIR:-.}/DockerData/dtm_db_data:/var/lib/postgresql/data/ env_file: .env networks: - dtm-network - container_name: db healthcheck: test: pg_isready -U ${POSTGRES_USER:-dtm} -d ${POSTGRES_DB:-dtm_db} start_period: 5s @@ -55,7 +53,7 @@ services: restart: always command: server /export --console-address 0.0.0.0:9090 --address 0.0.0.0:9000 volumes: - - ./DockerData/minio_data:/export + - ${PROJECT_DIR:-.}/DockerData/minio_data:/export environment: MINIO_ROOT_USER: ${S3_ACCESS_KEY:-dtm_user} MINIO_ROOT_PASSWORD: ${S3_SECRET_KEY:-somelongpassword} @@ -81,9 +79,8 @@ services: build: context: . dockerfile: src/backend/Dockerfile - container_name: dtm_migrations volumes: - - ./src/backend:/project/src/backend + - ${PROJECT_DIR:-.}/src/backend:/project/src/backend depends_on: - backend - db diff --git a/src/backend/Dockerfile b/src/backend/Dockerfile index e66ed6b5..89852656 100644 --- a/src/backend/Dockerfile +++ b/src/backend/Dockerfile @@ -46,6 +46,7 @@ RUN set -ex \ && DEBIAN_FRONTEND=noninteractive apt-get install \ -y --no-install-recommends \ "nano" \ + "curl" \ "gettext-base" \ "libpcre3" \ "libpq5" \ @@ -75,5 +76,6 @@ RUN pip install -U pdm COPY src /project/src # Set the entrypoint for the container -ENTRYPOINT ["uvicorn", "app.main:api", "--host", "0.0.0.0", "--port", "8000", \ +ENTRYPOINT [ "/project/src/backend/docker-entrypoint.sh" ] +CMD ["uvicorn", "app.main:api", "--host", "0.0.0.0", "--port", "8000", \ "--workers", "4", "--log-level", "info", "--reload"] diff --git a/src/backend/docker-entrypoint.sh b/src/backend/docker-entrypoint.sh new file mode 100755 index 00000000..67a9eec2 --- /dev/null +++ b/src/backend/docker-entrypoint.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +set -eo pipefail + +wait_for_db() { + max_retries=30 + retry_interval=5 + + for ((i = 0; i < max_retries; i++)); do + if /dev/null; then + echo "MINIO is available." + return 0 # Minio is available, exit successfully + fi + echo "Minio is not yet available. Retrying in ${retry_interval} seconds..." + sleep ${retry_interval} + done + + echo "Timed out waiting for Minio to become available." + exit 1 # Exit with an error code +} + +get_frontend_index_html() { + if [ ! -f /project/src/backend/templates/index.html ] + then + echo "/project/src/backend/templates/index.html file does not exist...trying to download from object storage.." + curl --fail --create-dirs ${S3_ENDPOINT}/${FRONTEND_BUCKET_NAME}/index.html --output /project/src/backend/templates/index.html || echo "Failed to download index.html... Please retry manually for now...." + else + echo "/project/src/backend/templates/index.html found. Continuing..." + fi +} + +# Start wait in background with tmp log files +wait_for_db & +wait_for_minio & +get_frontend_index_html & +wait + +exec "$@" + +exit 0 diff --git a/src/frontend/docker-entrypoint.sh b/src/frontend/docker-entrypoint.sh index cb5b7993..42fcc725 100755 --- a/src/frontend/docker-entrypoint.sh +++ b/src/frontend/docker-entrypoint.sh @@ -5,6 +5,7 @@ if [[ -z "${S3_ACCESS_KEY}" ]]; then echo "Missing environment variable S3_ACCESS_KEY" exit 0 fi + if [[ -z "${S3_SECRET_KEY}" ]]; then echo "Missing environment variable S3_SECRET_KEY" exit 0