From c186f439ba3c4d318c81cca110c7c69b63de2b0b Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Mon, 14 Oct 2024 17:49:57 +0500 Subject: [PATCH 1/7] feat: added dockerfile and docker image push workflow for xqueue --- .github/workflows/push-xqueue-image.yaml | 63 +++++++++++++++++++++++ dockerfiles/xqueue.Dockerfile | 65 ++++++++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 .github/workflows/push-xqueue-image.yaml create mode 100644 dockerfiles/xqueue.Dockerfile diff --git a/.github/workflows/push-xqueue-image.yaml b/.github/workflows/push-xqueue-image.yaml new file mode 100644 index 0000000..6a05215 --- /dev/null +++ b/.github/workflows/push-xqueue-image.yaml @@ -0,0 +1,63 @@ +name: Build and Push Xqueue Image + +on: + workflow_dispatch: + inputs: + branch: + description: "Target branch from which the source dockerfile from image will be sourced" + + schedule: + - cron: "0 4 * * 1-5" # UTC Time + +# Added for testing purposes. Will remove once the PR is finalised + pull_request: + branches: + - '**' + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + + steps: + - name: Get tag name + id: get-tag-name + uses: actions/github-script@v5 + with: + script: | + const tagName = "${{ github.event.inputs.branch }}" || 'latest'; + console.log('Will use tag: ' + tagName); + return tagName; + result-encoding: string + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build and push Dev Docker image + uses: docker/build-push-action@v6 + with: + file: ./dockerfiles/xqueue.Dockerfile + push: true + target: dev + tags: edxops/xqueue-dev-test:${{ steps.get-tag-name.outputs.result }} + + - name: Send failure notification + if: failure() + uses: dawidd6/action-send-mail@v3 + with: + server_address: email-smtp.us-east-1.amazonaws.com + server_port: 465 + username: ${{secrets.edx_smtp_username}} + password: ${{secrets.edx_smtp_password}} + subject: Push Image to docker.io/edxops failed in Xqueue + to: team-cosmonauts@edx.org + from: github-actions + body: Push Image to docker.io/edxops for Xqueue failed! For details see "github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" diff --git a/dockerfiles/xqueue.Dockerfile b/dockerfiles/xqueue.Dockerfile new file mode 100644 index 0000000..795706f --- /dev/null +++ b/dockerfiles/xqueue.Dockerfile @@ -0,0 +1,65 @@ +FROM ubuntu:focal as app + +# System requirements + +RUN apt-get update && \ + apt-get upgrade -qy && DEBIAN_FRONTEND=noninteractive apt-get install language-pack-en locales git \ + python3.8-dev python3-virtualenv libmysqlclient-dev libssl-dev build-essential pkg-config wget unzip -qy && \ + rm -rf /var/lib/apt/lists/* + +# Python is Python3. +RUN ln -s /usr/bin/python3 /usr/bin/python + +# Use UTF-8. +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + + +ARG COMMON_APP_DIR="/edx/app" +ARG XQUEUE_APP_DIR="${COMMON_APP_DIR}/xqueue" +ENV XQUEUE_APP_DIR="${COMMON_APP_DIR}/xqueue" +ENV XQUEUE_VENV_DIR="${COMMON_APP_DIR}/xqueue/venvs/xqueue" +ENV XQUEUE_CODE_DIR="${XQUEUE_APP_DIR}/xqueue" + +ENV PATH="$XQUEUE_VENV_DIR/bin:$PATH" + +# Working directory will be root of repo. +WORKDIR ${XQUEUE_CODE_DIR} + +# Install curl +RUN apt-get update && apt-get install -y curl +# cloning git repo +RUN curl -L https://github.com/openedx/xqueue/archive/refs/heads/master.tar.gz | tar -xz --strip-components=1 + +RUN virtualenv -p python3.8 --always-copy ${XQUEUE_VENV_DIR} + +# placeholder file for the time being unless devstack provisioning scripts need it. +RUN touch ${XQUEUE_APP_DIR}/xqueue_env + +# Expose ports. +EXPOSE 8040 + +FROM app as dev + +# xqueue service config commands below +RUN pip install -r ${XQUEUE_CODE_DIR}/requirements/dev.txt + +ENV DJANGO_SETTINGS_MODULE xqueue.devstack + +CMD while true; do python ./manage.py runserver 0.0.0.0:8040; sleep 2; done + +FROM app as production + +# xqueue service config commands below +RUN pip install -r ${XQUEUE_APP_DIR}/requirements.txt + +ENV DJANGO_SETTINGS_MODULE xqueue.production + +CMD gunicorn \ + --pythonpath=/edx/app/xqueue/xqueue \ + --timeout=300 \ + -b 0.0.0.0:8040 \ + -w 2 \ + - xqueue.wsgi:application From 608cd22161a167fb69235c32faca59367da0b5a8 Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Mon, 14 Oct 2024 17:55:09 +0500 Subject: [PATCH 2/7] chore: Remove pull_request trigger from workflow --- .github/workflows/push-xqueue-image.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/push-xqueue-image.yaml b/.github/workflows/push-xqueue-image.yaml index 6a05215..1c7c36f 100644 --- a/.github/workflows/push-xqueue-image.yaml +++ b/.github/workflows/push-xqueue-image.yaml @@ -9,11 +9,6 @@ on: schedule: - cron: "0 4 * * 1-5" # UTC Time -# Added for testing purposes. Will remove once the PR is finalised - pull_request: - branches: - - '**' - jobs: build-and-push-image: runs-on: ubuntu-latest @@ -47,7 +42,7 @@ jobs: file: ./dockerfiles/xqueue.Dockerfile push: true target: dev - tags: edxops/xqueue-dev-test:${{ steps.get-tag-name.outputs.result }} + tags: edxops/xqueue-dev:${{ steps.get-tag-name.outputs.result }} - name: Send failure notification if: failure() From 5dbe6ae64db09eed9e0abc69afe9c852fed2accf Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:15:54 +0500 Subject: [PATCH 3/7] chore: updated mail team for failure notification --- .github/workflows/push-xqueue-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-xqueue-image.yaml b/.github/workflows/push-xqueue-image.yaml index 1c7c36f..72cf5ac 100644 --- a/.github/workflows/push-xqueue-image.yaml +++ b/.github/workflows/push-xqueue-image.yaml @@ -53,6 +53,6 @@ jobs: username: ${{secrets.edx_smtp_username}} password: ${{secrets.edx_smtp_password}} subject: Push Image to docker.io/edxops failed in Xqueue - to: team-cosmonauts@edx.org + to: team-aurora@edx.org from: github-actions body: Push Image to docker.io/edxops for Xqueue failed! For details see "github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" From 67376e8a353ba759b4b8c7ad4eec20548f86d0e4 Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Wed, 16 Oct 2024 17:41:27 +0500 Subject: [PATCH 4/7] perf: updated Dockerfile to optimize requirements installation and dependency caching --- .github/workflows/push-xqueue-image.yaml | 5 +++++ dockerfiles/xqueue.Dockerfile | 12 ++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/push-xqueue-image.yaml b/.github/workflows/push-xqueue-image.yaml index 72cf5ac..1e99c87 100644 --- a/.github/workflows/push-xqueue-image.yaml +++ b/.github/workflows/push-xqueue-image.yaml @@ -9,6 +9,11 @@ on: schedule: - cron: "0 4 * * 1-5" # UTC Time +# Added for testing purposes. Will remove once the PR is finalised + pull_request: + branches: + - '**' + jobs: build-and-push-image: runs-on: ubuntu-latest diff --git a/dockerfiles/xqueue.Dockerfile b/dockerfiles/xqueue.Dockerfile index 795706f..ee4041a 100644 --- a/dockerfiles/xqueue.Dockerfile +++ b/dockerfiles/xqueue.Dockerfile @@ -3,7 +3,7 @@ FROM ubuntu:focal as app # System requirements RUN apt-get update && \ - apt-get upgrade -qy && DEBIAN_FRONTEND=noninteractive apt-get install language-pack-en locales git \ + apt-get upgrade -qy && DEBIAN_FRONTEND=noninteractive apt-get install language-pack-en locales git curl \ python3.8-dev python3-virtualenv libmysqlclient-dev libssl-dev build-essential pkg-config wget unzip -qy && \ rm -rf /var/lib/apt/lists/* @@ -16,7 +16,6 @@ ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 - ARG COMMON_APP_DIR="/edx/app" ARG XQUEUE_APP_DIR="${COMMON_APP_DIR}/xqueue" ENV XQUEUE_APP_DIR="${COMMON_APP_DIR}/xqueue" @@ -25,17 +24,16 @@ ENV XQUEUE_CODE_DIR="${XQUEUE_APP_DIR}/xqueue" ENV PATH="$XQUEUE_VENV_DIR/bin:$PATH" +RUN mkdir -p requirements + # Working directory will be root of repo. WORKDIR ${XQUEUE_CODE_DIR} - -# Install curl -RUN apt-get update && apt-get install -y curl # cloning git repo RUN curl -L https://github.com/openedx/xqueue/archive/refs/heads/master.tar.gz | tar -xz --strip-components=1 RUN virtualenv -p python3.8 --always-copy ${XQUEUE_VENV_DIR} -# placeholder file for the time being unless devstack provisioning scripts need it. +# Create placeholder file for devstack provisioning, if needed RUN touch ${XQUEUE_APP_DIR}/xqueue_env # Expose ports. @@ -43,6 +41,7 @@ EXPOSE 8040 FROM app as dev +RUN curl -L -o ${XQUEUE_CODE_DIR}/requirements/dev.txt https://raw.githubusercontent.com/openedx/xqueue/master/requirements/dev.txt # xqueue service config commands below RUN pip install -r ${XQUEUE_CODE_DIR}/requirements/dev.txt @@ -52,6 +51,7 @@ CMD while true; do python ./manage.py runserver 0.0.0.0:8040; sleep 2; done FROM app as production +RUN curl -L -o ${XQUEUE_APP_DIR}/requirements.txt https://raw.githubusercontent.com/openedx/xqueue/master/requirements.txt # xqueue service config commands below RUN pip install -r ${XQUEUE_APP_DIR}/requirements.txt From 67e19d03b1fb4c1c36134ac6f290e49cfedf1d59 Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Wed, 16 Oct 2024 17:44:22 +0500 Subject: [PATCH 5/7] chore: Remove pull_request trigger from workflow --- .github/workflows/push-xqueue-image.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/push-xqueue-image.yaml b/.github/workflows/push-xqueue-image.yaml index 1e99c87..72cf5ac 100644 --- a/.github/workflows/push-xqueue-image.yaml +++ b/.github/workflows/push-xqueue-image.yaml @@ -9,11 +9,6 @@ on: schedule: - cron: "0 4 * * 1-5" # UTC Time -# Added for testing purposes. Will remove once the PR is finalised - pull_request: - branches: - - '**' - jobs: build-and-push-image: runs-on: ubuntu-latest From 8fbbb6f0336539f332992b403ca9ca703c82a8f9 Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Fri, 18 Oct 2024 15:03:36 +0500 Subject: [PATCH 6/7] perf: updated Dockerfile to optimize requirements installation and dependency caching --- .github/workflows/push-xqueue-image.yaml | 5 +++++ dockerfiles/xqueue.Dockerfile | 12 ++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/push-xqueue-image.yaml b/.github/workflows/push-xqueue-image.yaml index 72cf5ac..1e99c87 100644 --- a/.github/workflows/push-xqueue-image.yaml +++ b/.github/workflows/push-xqueue-image.yaml @@ -9,6 +9,11 @@ on: schedule: - cron: "0 4 * * 1-5" # UTC Time +# Added for testing purposes. Will remove once the PR is finalised + pull_request: + branches: + - '**' + jobs: build-and-push-image: runs-on: ubuntu-latest diff --git a/dockerfiles/xqueue.Dockerfile b/dockerfiles/xqueue.Dockerfile index ee4041a..f62b4cd 100644 --- a/dockerfiles/xqueue.Dockerfile +++ b/dockerfiles/xqueue.Dockerfile @@ -24,12 +24,10 @@ ENV XQUEUE_CODE_DIR="${XQUEUE_APP_DIR}/xqueue" ENV PATH="$XQUEUE_VENV_DIR/bin:$PATH" -RUN mkdir -p requirements - # Working directory will be root of repo. WORKDIR ${XQUEUE_CODE_DIR} -# cloning git repo -RUN curl -L https://github.com/openedx/xqueue/archive/refs/heads/master.tar.gz | tar -xz --strip-components=1 + +RUN mkdir -p requirements RUN virtualenv -p python3.8 --always-copy ${XQUEUE_VENV_DIR} @@ -45,6 +43,9 @@ RUN curl -L -o ${XQUEUE_CODE_DIR}/requirements/dev.txt https://raw.githubusercon # xqueue service config commands below RUN pip install -r ${XQUEUE_CODE_DIR}/requirements/dev.txt +# cloning git repo +RUN curl -L https://github.com/openedx/xqueue/archive/refs/heads/master.tar.gz | tar -xz --strip-components=1 + ENV DJANGO_SETTINGS_MODULE xqueue.devstack CMD while true; do python ./manage.py runserver 0.0.0.0:8040; sleep 2; done @@ -55,6 +56,9 @@ RUN curl -L -o ${XQUEUE_APP_DIR}/requirements.txt https://raw.githubusercontent. # xqueue service config commands below RUN pip install -r ${XQUEUE_APP_DIR}/requirements.txt +# cloning git repo +RUN curl -L https://github.com/openedx/xqueue/archive/refs/heads/master.tar.gz | tar -xz --strip-components=1 + ENV DJANGO_SETTINGS_MODULE xqueue.production CMD gunicorn \ From 536d2094d0fe479617bf579de26db19088185a77 Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Fri, 18 Oct 2024 15:07:44 +0500 Subject: [PATCH 7/7] chore: Remove pull_request trigger from workflow --- .github/workflows/push-xqueue-image.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/push-xqueue-image.yaml b/.github/workflows/push-xqueue-image.yaml index 1e99c87..72cf5ac 100644 --- a/.github/workflows/push-xqueue-image.yaml +++ b/.github/workflows/push-xqueue-image.yaml @@ -9,11 +9,6 @@ on: schedule: - cron: "0 4 * * 1-5" # UTC Time -# Added for testing purposes. Will remove once the PR is finalised - pull_request: - branches: - - '**' - jobs: build-and-push-image: runs-on: ubuntu-latest