From e3dfcd6cf0b1c4b52d1c074f9e92775846984e94 Mon Sep 17 00:00:00 2001 From: Julian Arz Date: Tue, 28 Jun 2022 00:45:54 +0200 Subject: [PATCH 1/6] rewrote github actions workflow with docker build Combines base and release workflow, because artifacts of base workflow were unused. Instead, now always pushes to :latest when build succeeds. Uses raw docker commands without buildx, build-push-action only for pushing --- .github/workflows/labyrinth.yml | 105 +++++++++----------------------- .github/workflows/release.yml | 64 ------------------- docker/backend.Dockerfile | 6 +- web-client/Dockerfile | 9 ++- web-client/package.json | 2 +- 5 files changed, 41 insertions(+), 145 deletions(-) delete mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/labyrinth.yml b/.github/workflows/labyrinth.yml index 7aafb242..600a6a03 100644 --- a/.github/workflows/labyrinth.yml +++ b/.github/workflows/labyrinth.yml @@ -11,103 +11,54 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v2 - - name: Set up node - uses: actions/setup-node@v1 - with: - node-version: "16.x" - - name: Install dependencies - run: | - npm ci - name: Lint run: | - npm run lint + docker build --target lint . - name: Run tests run: | - npm run test:unit - - name: Build web-client - run: | - npm run build - - name: Archive web-client - uses: actions/upload-artifact@v2 - with: - name: webClient - path: | - dist/ - docker/ + docker build --target test . algolibs: runs-on: ubuntu-latest - defaults: - run: - working-directory: ./algolibs steps: - name: Check out repository uses: actions/checkout@v2 - - name: Setup cmake - uses: lukka/get-cmake@latest - - name: Run cmake and build shared library - run: | - export CC=gcc-9 - export CXX=g++-9 - mkdir builds - cmake -S. -Bbuilds/shared -DCMAKE_BUILD_TYPE=RELEASE - cmake --build builds/shared - mkdir -p ../backend/instance/lib/ - cp builds/shared/solvers/*.so ../backend/instance/lib/ - name: Run tests run: | - mkdir ../test-results - cd builds/shared/test/ - ctest -V - mv all_tests.xml ../../../../test-results/algolibs-results.xml - cd ../../../ - - name: Archive shared library - uses: actions/upload-artifact@v1 - with: - name: sharedLib - path: backend/instance/lib/ + docker build --target algolibs-test --file docker/backend.Dockerfile . backend: runs-on: ubuntu-latest - defaults: - run: - working-directory: ./backend - needs: [web-client, algolibs] + needs: [algolibs] steps: - name: Check out repository uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Install dependencies - run: | - python -m venv venv - . venv/bin/activate - pip install --upgrade pip - pip install -r dev-requirements.txt - name: Lint run: | - . venv/bin/activate - flake8 . --count --max-line-length=120 --max-complexity=10 --show-source --statistics --exclude __pycache__,venv - - name: Restore shared library artifact - uses: actions/download-artifact@v1 - with: - name: sharedLib - path: backend/instance/lib/ + docker build --target lint --file docker/backend.Dockerfile . - name: Run tests run: | - . venv/bin/activate - pytest . --junitxml=../test-results/backend-results.xml --cov=backend --cov-report=xml:../test-results/backend-coverage.xml - - name: Archive deployment - uses: actions/upload-artifact@v2 + docker build --target test --file docker/backend.Dockerfile . + release: + runs-on: ubuntu-latest + needs: [web-client, algolibs, backend] + steps: + - name: Log in to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Build and push web-client image + uses: docker/build-push-action@v2 + with: + context: ./web-client + file: ./web-client/Dockerfile + push: true + tags: julianarz/labyrinth-web-client:latest + - name: Build and push Docker image + uses: docker/build-push-action@v2 with: - name: deployment - path: | - backend/instance - backend/static - backend/labyrinth - backend/requirements.txt - backend/labyrinth_main.py - backend/uwsgi-docker.ini - backend/Dockerfile + context: ./ + file: ./docker/backend.Dockerfile + push: true + tags: julianarz/labyrinth-backend:latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 2dbf24f3..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: release - -on: - workflow_dispatch: - push: - tags: - '[0-9]+.[0-9]+.[0-9]+' - -jobs: - web-client: - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@v2 - - name: Log in to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v3 - with: - images: julianarz/labyrinth-web-client - tags: | - type=ref,priority=600,event=branch - type=match,priority=600,pattern=\d.\d.\d - type=raw,priority=200,value=latest - - name: Build and push Docker image - uses: docker/build-push-action@v2 - with: - context: ./web-client - file: ./web-client/Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - backend: - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@v2 - - name: Log in to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v3 - with: - images: julianarz/labyrinth-backend - tags: | - type=ref,priority=600,event=branch - type=match,priority=600,pattern=\d.\d.\d - type=raw,priority=200,value=latest - - name: Build and push Docker image - uses: docker/build-push-action@v2 - with: - context: ./ - file: ./docker/backend.Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/docker/backend.Dockerfile b/docker/backend.Dockerfile index 0f56d220..40dce728 100644 --- a/docker/backend.Dockerfile +++ b/docker/backend.Dockerfile @@ -45,14 +45,18 @@ COPY backend/instance ./instance RUN rm -rf instance/lib && mkdir instance/lib COPY --from=algolibs-release /usr/src/algolibs/build/solvers/*.so instance/lib/ -FROM base as test +FROM base as dev COPY backend/dev-requirements.txt . RUN pip install --no-cache-dir -r dev-requirements.txt +FROM dev as lint + # lint RUN flake8 . --count --max-line-length=120 --max-complexity=10 --show-source --statistics --exclude __pycache__,venv +FROM dev as test + # test RUN pytest . --junitxml=../test-results/backend-results.xml --cov=backend --cov-report=xml:../test-results/backend-coverage.xml diff --git a/web-client/Dockerfile b/web-client/Dockerfile index 07a6766e..3caf27aa 100644 --- a/web-client/Dockerfile +++ b/web-client/Dockerfile @@ -3,15 +3,20 @@ FROM node:16 as base WORKDIR /usr/src/web-client # copy everything but .dockerignore -COPY . ./ +COPY package.json package-lock.json ./ RUN npm ci +COPY . ./ + FROM base as test -RUN npm run lint RUN npm run test:unit +FROM base as lint + +RUN npm run lint + FROM base as build RUN npm run build diff --git a/web-client/package.json b/web-client/package.json index 6511d8d0..dcd885d8 100644 --- a/web-client/package.json +++ b/web-client/package.json @@ -6,7 +6,7 @@ "serve": "vue-cli-service serve", "build": "vue-cli-service build", "test:unit": "vue-cli-service test:unit --maxWorkers=4", - "lint": "vue-cli-service lint", + "lint": "vue-cli-service lint --no-fix", "debug-build": "vue-cli-service build --mode development" }, "dependencies": { From 2ad690f1b6dfe98392978761654814b2c032eece Mon Sep 17 00:00:00 2001 From: Julian Arz Date: Tue, 28 Jun 2022 01:51:24 +0200 Subject: [PATCH 2/6] fixes backend test inside container coverage is broken currently, but not used anyways --- docker/backend.Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/backend.Dockerfile b/docker/backend.Dockerfile index 40dce728..56f394e8 100644 --- a/docker/backend.Dockerfile +++ b/docker/backend.Dockerfile @@ -50,15 +50,15 @@ FROM base as dev COPY backend/dev-requirements.txt . RUN pip install --no-cache-dir -r dev-requirements.txt -FROM dev as lint - # lint +FROM dev as lint RUN flake8 . --count --max-line-length=120 --max-complexity=10 --show-source --statistics --exclude __pycache__,venv -FROM dev as test - # test -RUN pytest . --junitxml=../test-results/backend-results.xml --cov=backend --cov-report=xml:../test-results/backend-coverage.xml +FROM dev as test +COPY backend/pytest.ini backend/conftest.py ./ +COPY backend/tests ./tests +RUN pytest . FROM base as release From fe99fa26980e85f2e857481882d71ae9851d0d53 Mon Sep 17 00:00:00 2001 From: Julian Arz Date: Tue, 28 Jun 2022 22:41:02 +0200 Subject: [PATCH 3/6] fixes release stage by adding checkout --- .github/workflows/labyrinth.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/labyrinth.yml b/.github/workflows/labyrinth.yml index 600a6a03..d644ce8d 100644 --- a/.github/workflows/labyrinth.yml +++ b/.github/workflows/labyrinth.yml @@ -43,6 +43,8 @@ jobs: runs-on: ubuntu-latest needs: [web-client, algolibs, backend] steps: + - name: Check out repository + uses: actions/checkout@v2 - name: Log in to Docker Hub uses: docker/login-action@v1 with: From cec1b7c1a1b9188ce582884ae033226a5e971b77 Mon Sep 17 00:00:00 2001 From: Julian Arz Date: Tue, 28 Jun 2022 23:58:26 +0200 Subject: [PATCH 4/6] sets up buildx in each job Currently, docker build in workflows does not respect multistage builds. Locally, it works correctly and omits unneeded stages. Supposedly because buildx is not the default builder in workflows. --- .github/workflows/labyrinth.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/labyrinth.yml b/.github/workflows/labyrinth.yml index d644ce8d..0a5c89ec 100644 --- a/.github/workflows/labyrinth.yml +++ b/.github/workflows/labyrinth.yml @@ -11,6 +11,10 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + install: true - name: Lint run: | docker build --target lint . @@ -23,6 +27,10 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + install: true - name: Run tests run: | docker build --target algolibs-test --file docker/backend.Dockerfile . @@ -33,6 +41,10 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + install: true - name: Lint run: | docker build --target lint --file docker/backend.Dockerfile . @@ -45,6 +57,10 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + install: true - name: Log in to Docker Hub uses: docker/login-action@v1 with: @@ -57,7 +73,7 @@ jobs: file: ./web-client/Dockerfile push: true tags: julianarz/labyrinth-web-client:latest - - name: Build and push Docker image + - name: Build and push backend image uses: docker/build-push-action@v2 with: context: ./ From 1a29cda43d22d973f65927316529fe5f17fc8ba4 Mon Sep 17 00:00:00 2001 From: Julian Arz Date: Wed, 29 Jun 2022 00:59:13 +0200 Subject: [PATCH 5/6] tries to speed up build by caching image layers --- .github/workflows/labyrinth.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/labyrinth.yml b/.github/workflows/labyrinth.yml index 0a5c89ec..c9c894e5 100644 --- a/.github/workflows/labyrinth.yml +++ b/.github/workflows/labyrinth.yml @@ -15,6 +15,13 @@ jobs: uses: docker/setup-buildx-action@v2 with: install: true + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- - name: Lint run: | docker build --target lint . @@ -31,6 +38,13 @@ jobs: uses: docker/setup-buildx-action@v2 with: install: true + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- - name: Run tests run: | docker build --target algolibs-test --file docker/backend.Dockerfile . @@ -45,6 +59,13 @@ jobs: uses: docker/setup-buildx-action@v2 with: install: true + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- - name: Lint run: | docker build --target lint --file docker/backend.Dockerfile . @@ -61,6 +82,13 @@ jobs: uses: docker/setup-buildx-action@v2 with: install: true + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- - name: Log in to Docker Hub uses: docker/login-action@v1 with: From 7a340988ef038d38ae53d7023f88cf2b80949bc7 Mon Sep 17 00:00:00 2001 From: Julian Arz Date: Wed, 29 Jun 2022 01:35:50 +0200 Subject: [PATCH 6/6] reverts docker cache it did not speed up the build. Apparently, it has to be used in combination with build-push-action. I currently prefer to use raw docker build commands for simplicity. --- .github/workflows/labyrinth.yml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/.github/workflows/labyrinth.yml b/.github/workflows/labyrinth.yml index c9c894e5..0a5c89ec 100644 --- a/.github/workflows/labyrinth.yml +++ b/.github/workflows/labyrinth.yml @@ -15,13 +15,6 @@ jobs: uses: docker/setup-buildx-action@v2 with: install: true - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - name: Lint run: | docker build --target lint . @@ -38,13 +31,6 @@ jobs: uses: docker/setup-buildx-action@v2 with: install: true - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - name: Run tests run: | docker build --target algolibs-test --file docker/backend.Dockerfile . @@ -59,13 +45,6 @@ jobs: uses: docker/setup-buildx-action@v2 with: install: true - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - name: Lint run: | docker build --target lint --file docker/backend.Dockerfile . @@ -82,13 +61,6 @@ jobs: uses: docker/setup-buildx-action@v2 with: install: true - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - name: Log in to Docker Hub uses: docker/login-action@v1 with: