From eb3a1e44047b2466da7fa4acef1aca3fde9fc694 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Tue, 8 Oct 2024 09:20:27 +0200 Subject: [PATCH 1/7] manage dependencies --- .docker/entrypoint.sh | 2 +- .github/workflows/dependencies.yml | 52 ++++++++++++++++++++++++++++++ .github/workflows/test.yml | 12 +++++-- Dockerfile | 18 +++++------ Makefile | 5 ++- docker-compose.yml | 2 +- 6 files changed, 77 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/dependencies.yml diff --git a/.docker/entrypoint.sh b/.docker/entrypoint.sh index d450269a..07fd2fed 100755 --- a/.docker/entrypoint.sh +++ b/.docker/entrypoint.sh @@ -3,7 +3,7 @@ cd /app/src || exit # Activate venv -. /app/venv/bin/activate +. /opt/venv/bin/activate if [ "$COLLECTSTATIC" == "1" ] then diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml new file mode 100644 index 00000000..eb8f4aae --- /dev/null +++ b/.github/workflows/dependencies.yml @@ -0,0 +1,52 @@ +name: Check deps + +on: + pull_request: + paths: + - setup.py + - requirements.txt + - requirements-dev.txt + +env: + DEBIAN_FRONTEND: noninteractive + +jobs: + quality: + name: Checking dependency graph + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ['ubuntu-20.04'] + python-version: ['3.8'] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python:v4 + with: + cache: pip + + - name: Install dependencies + run: | + pip3 install -c requirements-dev.txt pip-tools + + - name: Check dependency graph + run: | + pip-compile -q --strip-extras + pip-compile -q --strip-extras requirements-dev.in + + - name: Verify dependency graph is ok + uses: tj-actions/verify-changed-files@v20 + id: verify-changed-files + with: + files: | + requirements.txt + requirements-dev.txt + + - name: Validating graph + if: steps.verify-changed-files.outputs.files_changed == 'true' + run: | + echo "Dependency file(s) changed: ${{ steps.verify-changed-files.outputs.changed_files }}" + git diff + core.setFailed('Please add your new dependencies in setup.py and/or dev-requirements.in then run pip-compile to add them in requirements. (see docs/contribute/development)') diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 73fcb8b4..1f600725 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -77,6 +77,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Install dependencies run: | sudo apt-get update -q @@ -85,11 +86,13 @@ jobs: pip install -r requirements.txt -U pip install -r requirements-dev.txt npm ci + - name: Test with coverage run: | cd src coverage run ./manage.py test coverage run -a ./manage.py test --settings screamshotter.settings.test_timeout screenshotter.tests.CaptureTestCase.test_timeout_screenshot + - name: Coverage upload run: | pip install codecov @@ -101,14 +104,17 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 + - name: Build Docker image run: | docker build -t screamshotter_ci:latest . + - name: Upload image - uses: ishworkh/docker-image-artifact-upload@v1 + uses: ishworkh/docker-image-artifact-upload@v2 with: image: "screamshotter_ci:latest" @@ -123,6 +129,7 @@ jobs: version: '20.04' - os: jammy version: '22.04' + env: DISTRO: 'ubuntu:${{ matrix.os }}' CODE: ${{ matrix.version }} @@ -150,9 +157,10 @@ jobs: name: Tests E2E docker runs-on: ubuntu-latest needs: [build_docker_image] + steps: - name: Download image - uses: ishworkh/docker-image-artifact-download@v1 + uses: ishworkh/docker-image-artifact-download@v2 with: image: "screamshotter_ci:latest" diff --git a/Dockerfile b/Dockerfile index 91e8f2eb..f962ae8b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,8 +13,8 @@ ENV MAX_REQUESTS=250 ENV PUPPETEER_CACHE_DIR=/app/puppeteer/ RUN useradd -ms /bin/bash django -RUN mkdir -p /app/static -RUN chown django:django /app +RUN mkdir -p /app/static /opt +RUN chown django:django /app /opt RUN apt-get -qq update && apt-get install -qq -y \ gconf-service \ @@ -81,28 +81,28 @@ RUN apt-get -qq update && apt-get install -qq -y \ RUN wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && rm get-pip.py USER django -RUN python3 -m venv /app/venv -RUN /app/venv/bin/pip3 install --no-cache-dir pip setuptools wheel -U +RUN python3 -m venv /opt/venv +RUN /opt/venv/bin/pip3 install --no-cache-dir pip setuptools wheel -U COPY requirements.txt /app/ -RUN /app/venv/bin/pip3 install --no-cache-dir -r /app/requirements.txt -U && rm /app/requirements.txt -RUN /app/venv/bin/nodeenv /app/venv/ -C '' -p -n 20.9.0 +RUN /opt/venv/bin/pip3 install --no-cache-dir -r /app/requirements.txt -U && rm /app/requirements.txt +RUN /opt/venv/bin/nodeenv /app/venv/ -C '' -p -n 20.9.0 # upgrade npm & requirements COPY package.json /app/package.json COPY package-lock.json /app/package-lock.json -RUN . /app/venv/bin/activate && npm ci && rm /app/*.json +RUN . /opt/venv/bin/activate && npm ci && rm /app/*.json FROM build AS dev COPY requirements-dev.txt /app/ -RUN /app/venv/bin/pip3 install --no-cache-dir -r /app/requirements-dev.txt -U && rm /app/requirements-dev.txt +RUN /opt/venv/bin/pip3 install --no-cache-dir -r /app/requirements-dev.txt -U && rm /app/requirements-dev.txt CMD ["./manage.py", "runserver", "0.0.0.0:8000"] FROM base -COPY --from=build /app/venv /app/venv +COPY --from=build /opt/venv /opt/venv COPY --from=build /app/node_modules /app/node_modules COPY --from=build /app/puppeteer /app/puppeteer COPY src /app/src diff --git a/Makefile b/Makefile index 4847a0e6..e2792851 100644 --- a/Makefile +++ b/Makefile @@ -4,4 +4,7 @@ build_deb: docker run --name screamshotter_deb_run -t screamshotter_deb bash -c "exit" docker cp screamshotter_deb_run:/dpkg ./ docker stop screamshotter_deb_run - docker rm screamshotter_deb_run \ No newline at end of file + docker rm screamshotter_deb_run + +deps: + docker compose run --rm web bash -c "pip-compile -q --strip-extras && pip-compile -q --strip-extras requirements-dev.in" diff --git a/docker-compose.yml b/docker-compose.yml index 34576273..0c095885 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ services: BASE_IMAGE: focal volumes: - - ./src/:/app/src/ + - ./:/app/ ports: - "8000:8000" environment: From 87a4726c31c57ad768c8c717d50dc8c254369dcf Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Tue, 8 Oct 2024 09:25:00 +0200 Subject: [PATCH 2/7] manage dependencies --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1f600725..897fbc81 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -114,7 +114,7 @@ jobs: docker build -t screamshotter_ci:latest . - name: Upload image - uses: ishworkh/docker-image-artifact-upload@v2 + uses: ishworkh/docker-image-artifact-upload@v2.1.0 with: image: "screamshotter_ci:latest" @@ -160,7 +160,7 @@ jobs: steps: - name: Download image - uses: ishworkh/docker-image-artifact-download@v2 + uses: ishworkh/docker-image-artifact-download@v2.1.0 with: image: "screamshotter_ci:latest" @@ -256,7 +256,7 @@ jobs: name: 'debian-20.04' - name: Download docker image - uses: ishworkh/docker-image-artifact-download@v1 + uses: ishworkh/docker-image-artifact-download@v2.1.0 with: image: "screamshotter_ci:latest" From f451ec02a4c56ddcfdc19280fa24f666dc7f1d50 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Tue, 8 Oct 2024 09:50:34 +0200 Subject: [PATCH 3/7] manage dependencies --- .github/workflows/dependencies.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index eb8f4aae..14f5a62a 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -22,10 +22,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python:v4 + - uses: actions/setup-python@v5 with: - cache: pip + cache: 'pip' + python-version: '3.8.18' - name: Install dependencies run: | From bdcfdb85c1a6adc4c7caa4e1de465666d200596e Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Tue, 8 Oct 2024 09:54:35 +0200 Subject: [PATCH 4/7] manage dependencies --- .github/workflows/dependencies.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 14f5a62a..913b1a6d 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -2,10 +2,10 @@ name: Check deps on: pull_request: - paths: - - setup.py - - requirements.txt - - requirements-dev.txt + #paths: + # - setup.py + # - requirements.txt + # - requirements-dev.txt env: DEBIAN_FRONTEND: noninteractive From 4ce9a762c9623ac16e32f827b0d1412756cb67cf Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Tue, 8 Oct 2024 10:10:58 +0200 Subject: [PATCH 5/7] fix dependencies --- .github/workflows/dependencies.yml | 8 ++++---- Makefile | 2 +- requirements-dev.txt | 4 ++-- requirements.txt | 7 +++---- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 913b1a6d..14f5a62a 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -2,10 +2,10 @@ name: Check deps on: pull_request: - #paths: - # - setup.py - # - requirements.txt - # - requirements-dev.txt + paths: + - setup.py + - requirements.txt + - requirements-dev.txt env: DEBIAN_FRONTEND: noninteractive diff --git a/Makefile b/Makefile index e2792851..b7ab79d7 100644 --- a/Makefile +++ b/Makefile @@ -7,4 +7,4 @@ build_deb: docker rm screamshotter_deb_run deps: - docker compose run --rm web bash -c "pip-compile -q --strip-extras && pip-compile -q --strip-extras requirements-dev.in" + docker compose run --rm web bash -c "cd /app && pip-compile -q --strip-extras && pip-compile -q --strip-extras requirements-dev.in" diff --git a/requirements-dev.txt b/requirements-dev.txt index 391b8a13..5f85157f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile requirements-dev.in +# pip-compile --strip-extras requirements-dev.in # asgiref==3.8.1 # via @@ -18,7 +18,7 @@ click==8.1.7 # via pip-tools coverage==7.6.1 # via -r requirements-dev.in -django==4.2.13 +django==4.2.16 # via # -c ./requirements.txt # django-debug-toolbar diff --git a/requirements.txt b/requirements.txt index 7115f3f7..1d662efc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,13 +2,14 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile +# pip-compile --strip-extras # asgiref==3.8.1 # via django backports-zoneinfo==0.2.1 ; python_version < "3.9" # via # django + # djangorestframework # screamshotter (setup.py) certifi==2024.8.30 # via @@ -30,7 +31,7 @@ gevent==24.2.1 # via gunicorn greenlet==3.1.1 # via gevent -gunicorn[gevent]==23.0.0 +gunicorn==23.0.0 # via screamshotter (setup.py) idna==3.7 # via requests @@ -44,8 +45,6 @@ nodeenv==1.9.1 # via screamshotter (setup.py) packaging==24.1 # via gunicorn -pytz==2024.2 - # via djangorestframework requests==2.32.3 # via coreapi sentry-sdk==2.15.0 From 3700da2070fafc9be011d8bf265ec11a0ba67bee Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Tue, 8 Oct 2024 10:59:47 +0200 Subject: [PATCH 6/7] fix tests --- src/screenshotter/tests.py | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/src/screenshotter/tests.py b/src/screenshotter/tests.py index 5be325bc..98a157b2 100644 --- a/src/screenshotter/tests.py +++ b/src/screenshotter/tests.py @@ -10,7 +10,7 @@ from django.test import SimpleTestCase, override_settings from django.urls import reverse from rest_framework.serializers import Serializer -from rest_framework.test import APIClient +from rest_framework.test import APISimpleTestCase from .exceptions import ScreenshotterException from .puppeteer import take_screenshot @@ -24,26 +24,22 @@ class CaptureTestCase(SimpleTestCase): @override_settings(MEDIA_ROOT=temp_dir.name) def test_capture_mime(self): png = take_screenshot('https://www.google.fr') - png_path = os.path.join(temp_dir.name, 'test.png') cfile = ContentFile(content=png) - default_storage.save(name=png_path, content=cfile) + default_storage.save(name='test_capture_mime.png', content=cfile) - self.assertTrue(os.path.exists(png_path)) - self.assertNotEqual(os.path.getsize(png_path), 0) - - mime = magic.from_file(png_path, mime=True) + self.assertNotEqual(cfile.size, 0) + mime = magic.from_buffer(default_storage.open('test_capture_mime.png').read(), mime=True) self.assertEqual(mime, "image/png", ) @override_settings(MEDIA_ROOT=temp_dir.name) def test_capture_size(self): png = take_screenshot('https://www.google.fr', width=1280, height=720) - png_path = os.path.join(temp_dir.name, 'test2.png') cfile = ContentFile(content=png) - default_storage.save(name=png_path, content=cfile) + default_storage.save(name='test_capture_size.png', content=cfile) - image = Image.open(png_path) + image = Image.open(default_storage.path('test_capture_size.png')) self.assertEqual(image.width, 1280) self.assertEqual(image.height, 720) @@ -78,23 +74,20 @@ def test_bad_settings(self): bool(app_settings.BAD_SETTINGS) -class CaptureApiTestCase(SimpleTestCase): - def setUp(self): - self.api_client = APIClient() - +class CaptureApiTestCase(APISimpleTestCase): def test_api_good_request_json(self): serializer = ScreenshotSerializer() data = serializer.data data['url'] = "https://www.google.fr" - response = self.api_client.post(reverse('screenshotter:screenshot') + '?format=json', data=data) + response = self.client.post(reverse('screenshotter:screenshot') + '?format=json', data=data) data = response.json() self.assertEqual(response.status_code, 200, data) self.assertIn('base64', data) def test_api_bad_request(self): serializer = ScreenshotSerializer() - response = self.api_client.post(reverse('screenshotter:screenshot') + '?format=json', data=serializer.data) + response = self.client.post(reverse('screenshotter:screenshot') + '?format=json', data=serializer.data) data = response.json() self.assertEqual(response.status_code, 400) self.assertIn('url', data) @@ -104,7 +97,7 @@ def test_api_wrong_response(self): serializer = ScreenshotSerializer() data = serializer.data data['url'] = "https://kikou.com" - response = self.api_client.post(reverse('screenshotter:screenshot') + '?format=json', data=data) + response = self.client.post(reverse('screenshotter:screenshot') + '?format=json', data=data) self.assertEqual(response.status_code, 500, response.json()) def test_api_browsable(self): @@ -112,7 +105,7 @@ def test_api_browsable(self): data = serializer.data data['url'] = "https://www.google.fr" - response = self.api_client.post(reverse('screenshotter:screenshot') + '?format=api', data=data) + response = self.client.post(reverse('screenshotter:screenshot') + '?format=api', data=data) self.assertEqual(response.status_code, 200) self.assertIn(b'', response.content) # check default renderer is json in browsable api (response in "base64": "xxx" format) @@ -123,7 +116,7 @@ def test_png_default(self): data = serializer.data data['url'] = "https://www.google.fr" - response = self.api_client.post(reverse('screenshotter:screenshot'), data=data) + response = self.client.post(reverse('screenshotter:screenshot'), data=data) self.assertEqual(response.status_code, 200) mime = magic.from_buffer(response.content, mime=True) self.assertEqual(mime, "image/png") From 9e24618e113928a5c623d24983d8816bede32d7f Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Tue, 8 Oct 2024 11:11:41 +0200 Subject: [PATCH 7/7] fix tests --- .github/workflows/test.yml | 1 + src/screenshotter/tests.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 897fbc81..bfe2e39a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -77,6 +77,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: pip - name: Install dependencies run: | diff --git a/src/screenshotter/tests.py b/src/screenshotter/tests.py index 98a157b2..ca0ce01f 100644 --- a/src/screenshotter/tests.py +++ b/src/screenshotter/tests.py @@ -1,4 +1,3 @@ -import os from tempfile import TemporaryDirectory from unittest import skipIf