From a5e9a37b79b4306fd7b588d519dd829dde43e361 Mon Sep 17 00:00:00 2001 From: Sebastian Echeverria Date: Wed, 20 Nov 2024 18:05:13 -0500 Subject: [PATCH 01/14] Initial CI setup, not fully tested yet --- .github/workflows/ci.yaml | 36 ++++++++++++ Makefile | 55 ++++++++++++++++++ poetry.lock | 115 +++++++++++++++++++++++++++++++------- pyproject.toml | 14 ++++- ruff.toml | 2 +- 5 files changed, 199 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/ci.yaml create mode 100644 Makefile diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..6409456 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,36 @@ +name: Continuous Integration + +on: + - push + - pull_request + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install poetry + run: curl -sSL https://install.python-poetry.org | python - + - name: Install dependencies + run: poetry install --with extra_dependencies + - name: Check format + run: make check-format + - name: Lint code + run: make check-lint + - name: Check types + run: make typecheck + - name: Execute unit tests + run: make test + + docker: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Docker Build Action + run: docker build -t vessel . diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..32af44e --- /dev/null +++ b/Makefile @@ -0,0 +1,55 @@ +# Automation of various common tasks + +# ----------------------------------------------------------------------------- +# QA +# ----------------------------------------------------------------------------- + +# Format all source code +.PHONY: format +format: + poetry run ruff format + +.PHONY: check-format +check-format: + poetry run ruff format --check + +# Lint all source code +.PHONY: lint +lint: + poetry run ruff check --fix + +.PHONY: check-lint +check-lint: + poetry run ruff check + +# Typecheck all source code +.PHONY: typecheck +typecheck: + poetry run mypy vessel/ + +.PHONY: check-typecheck +check-typecheck: typecheck + +# Clean cache files +.PHONY: clean +clean: + rm -r -f .mypy_cache .pytest_cache .ruff_cache + +# All quality assurance +.PHONY: qa +qa: format lint typecheck + +# Check all QA tasks +.PHONY: check +check: check-lint check-typecheck + +# Run unit tests with pytest +.PHONY: test +test: + poetry run pytest test + +# ----------------------------------------------------------------------------- +# All actions and checks needed to update and review for pushing. +# ----------------------------------------------------------------------------- +.PHONY: ci +ci: clean qa test diff --git a/poetry.lock b/poetry.lock index db874b8..9ab6977 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "alembic" @@ -1085,6 +1085,69 @@ files = [ [package.extras] test = ["pytest", "pytest-benchmark"] +[[package]] +name = "mypy" +version = "1.13.0" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-1.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6607e0f1dd1fb7f0aca14d936d13fd19eba5e17e1cd2a14f808fa5f8f6d8f60a"}, + {file = "mypy-1.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8a21be69bd26fa81b1f80a61ee7ab05b076c674d9b18fb56239d72e21d9f4c80"}, + {file = "mypy-1.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b2353a44d2179846a096e25691d54d59904559f4232519d420d64da6828a3a7"}, + {file = "mypy-1.13.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0730d1c6a2739d4511dc4253f8274cdd140c55c32dfb0a4cf8b7a43f40abfa6f"}, + {file = "mypy-1.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:c5fc54dbb712ff5e5a0fca797e6e0aa25726c7e72c6a5850cfd2adbc1eb0a372"}, + {file = "mypy-1.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:581665e6f3a8a9078f28d5502f4c334c0c8d802ef55ea0e7276a6e409bc0d82d"}, + {file = "mypy-1.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3ddb5b9bf82e05cc9a627e84707b528e5c7caaa1c55c69e175abb15a761cec2d"}, + {file = "mypy-1.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20c7ee0bc0d5a9595c46f38beb04201f2620065a93755704e141fcac9f59db2b"}, + {file = "mypy-1.13.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3790ded76f0b34bc9c8ba4def8f919dd6a46db0f5a6610fb994fe8efdd447f73"}, + {file = "mypy-1.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:51f869f4b6b538229c1d1bcc1dd7d119817206e2bc54e8e374b3dfa202defcca"}, + {file = "mypy-1.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5c7051a3461ae84dfb5dd15eff5094640c61c5f22257c8b766794e6dd85e72d5"}, + {file = "mypy-1.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:39bb21c69a5d6342f4ce526e4584bc5c197fd20a60d14a8624d8743fffb9472e"}, + {file = "mypy-1.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:164f28cb9d6367439031f4c81e84d3ccaa1e19232d9d05d37cb0bd880d3f93c2"}, + {file = "mypy-1.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a4c1bfcdbce96ff5d96fc9b08e3831acb30dc44ab02671eca5953eadad07d6d0"}, + {file = "mypy-1.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0affb3a79a256b4183ba09811e3577c5163ed06685e4d4b46429a271ba174d2"}, + {file = "mypy-1.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a7b44178c9760ce1a43f544e595d35ed61ac2c3de306599fa59b38a6048e1aa7"}, + {file = "mypy-1.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5d5092efb8516d08440e36626f0153b5006d4088c1d663d88bf79625af3d1d62"}, + {file = "mypy-1.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2904956dac40ced10931ac967ae63c5089bd498542194b436eb097a9f77bc8"}, + {file = "mypy-1.13.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:7bfd8836970d33c2105562650656b6846149374dc8ed77d98424b40b09340ba7"}, + {file = "mypy-1.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:9f73dba9ec77acb86457a8fc04b5239822df0c14a082564737833d2963677dbc"}, + {file = "mypy-1.13.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:100fac22ce82925f676a734af0db922ecfea991e1d7ec0ceb1e115ebe501301a"}, + {file = "mypy-1.13.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7bcb0bb7f42a978bb323a7c88f1081d1b5dee77ca86f4100735a6f541299d8fb"}, + {file = "mypy-1.13.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bde31fc887c213e223bbfc34328070996061b0833b0a4cfec53745ed61f3519b"}, + {file = "mypy-1.13.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:07de989f89786f62b937851295ed62e51774722e5444a27cecca993fc3f9cd74"}, + {file = "mypy-1.13.0-cp38-cp38-win_amd64.whl", hash = "sha256:4bde84334fbe19bad704b3f5b78c4abd35ff1026f8ba72b29de70dda0916beb6"}, + {file = "mypy-1.13.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0246bcb1b5de7f08f2826451abd947bf656945209b140d16ed317f65a17dc7dc"}, + {file = "mypy-1.13.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7f5b7deae912cf8b77e990b9280f170381fdfbddf61b4ef80927edd813163732"}, + {file = "mypy-1.13.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7029881ec6ffb8bc233a4fa364736789582c738217b133f1b55967115288a2bc"}, + {file = "mypy-1.13.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3e38b980e5681f28f033f3be86b099a247b13c491f14bb8b1e1e134d23bb599d"}, + {file = "mypy-1.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:a6789be98a2017c912ae6ccb77ea553bbaf13d27605d2ca20a76dfbced631b24"}, + {file = "mypy-1.13.0-py3-none-any.whl", hash = "sha256:9c250883f9fd81d212e0952c92dbfcc96fc237f4b7c92f56ac81fd48460b3e5a"}, + {file = "mypy-1.13.0.tar.gz", hash = "sha256:0291a61b6fbf3e6673e3405cfcc0e7650bebc7939659fdca2702958038bd835e"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +typing-extensions = ">=4.6.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +faster-cache = ["orjson"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + [[package]] name = "networkx" version = "3.4.2" @@ -1598,28 +1661,29 @@ testing = ["tox"] [[package]] name = "ruff" -version = "0.3.7" +version = "0.7.4" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.3.7-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:0e8377cccb2f07abd25e84fc5b2cbe48eeb0fea9f1719cad7caedb061d70e5ce"}, - {file = "ruff-0.3.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:15a4d1cc1e64e556fa0d67bfd388fed416b7f3b26d5d1c3e7d192c897e39ba4b"}, - {file = "ruff-0.3.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d28bdf3d7dc71dd46929fafeec98ba89b7c3550c3f0978e36389b5631b793663"}, - {file = "ruff-0.3.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:379b67d4f49774ba679593b232dcd90d9e10f04d96e3c8ce4a28037ae473f7bb"}, - {file = "ruff-0.3.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c060aea8ad5ef21cdfbbe05475ab5104ce7827b639a78dd55383a6e9895b7c51"}, - {file = "ruff-0.3.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:ebf8f615dde968272d70502c083ebf963b6781aacd3079081e03b32adfe4d58a"}, - {file = "ruff-0.3.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d48098bd8f5c38897b03604f5428901b65e3c97d40b3952e38637b5404b739a2"}, - {file = "ruff-0.3.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da8a4fda219bf9024692b1bc68c9cff4b80507879ada8769dc7e985755d662ea"}, - {file = "ruff-0.3.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c44e0149f1d8b48c4d5c33d88c677a4aa22fd09b1683d6a7ff55b816b5d074f"}, - {file = "ruff-0.3.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3050ec0af72b709a62ecc2aca941b9cd479a7bf2b36cc4562f0033d688e44fa1"}, - {file = "ruff-0.3.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a29cc38e4c1ab00da18a3f6777f8b50099d73326981bb7d182e54a9a21bb4ff7"}, - {file = "ruff-0.3.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5b15cc59c19edca917f51b1956637db47e200b0fc5e6e1878233d3a938384b0b"}, - {file = "ruff-0.3.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e491045781b1e38b72c91247cf4634f040f8d0cb3e6d3d64d38dcf43616650b4"}, - {file = "ruff-0.3.7-py3-none-win32.whl", hash = "sha256:bc931de87593d64fad3a22e201e55ad76271f1d5bfc44e1a1887edd0903c7d9f"}, - {file = "ruff-0.3.7-py3-none-win_amd64.whl", hash = "sha256:5ef0e501e1e39f35e03c2acb1d1238c595b8bb36cf7a170e7c1df1b73da00e74"}, - {file = "ruff-0.3.7-py3-none-win_arm64.whl", hash = "sha256:789e144f6dc7019d1f92a812891c645274ed08af6037d11fc65fcbc183b7d59f"}, - {file = "ruff-0.3.7.tar.gz", hash = "sha256:d5c1aebee5162c2226784800ae031f660c350e7a3402c4d1f8ea4e97e232e3ba"}, + {file = "ruff-0.7.4-py3-none-linux_armv6l.whl", hash = "sha256:a4919925e7684a3f18e18243cd6bea7cfb8e968a6eaa8437971f681b7ec51478"}, + {file = "ruff-0.7.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:cfb365c135b830778dda8c04fb7d4280ed0b984e1aec27f574445231e20d6c63"}, + {file = "ruff-0.7.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:63a569b36bc66fbadec5beaa539dd81e0527cb258b94e29e0531ce41bacc1f20"}, + {file = "ruff-0.7.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d06218747d361d06fd2fdac734e7fa92df36df93035db3dc2ad7aa9852cb109"}, + {file = "ruff-0.7.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e0cea28d0944f74ebc33e9f934238f15c758841f9f5edd180b5315c203293452"}, + {file = "ruff-0.7.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80094ecd4793c68b2571b128f91754d60f692d64bc0d7272ec9197fdd09bf9ea"}, + {file = "ruff-0.7.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:997512325c6620d1c4c2b15db49ef59543ef9cd0f4aa8065ec2ae5103cedc7e7"}, + {file = "ruff-0.7.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:00b4cf3a6b5fad6d1a66e7574d78956bbd09abfd6c8a997798f01f5da3d46a05"}, + {file = "ruff-0.7.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7dbdc7d8274e1422722933d1edddfdc65b4336abf0b16dfcb9dedd6e6a517d06"}, + {file = "ruff-0.7.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e92dfb5f00eaedb1501b2f906ccabfd67b2355bdf117fea9719fc99ac2145bc"}, + {file = "ruff-0.7.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3bd726099f277d735dc38900b6a8d6cf070f80828877941983a57bca1cd92172"}, + {file = "ruff-0.7.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:2e32829c429dd081ee5ba39aef436603e5b22335c3d3fff013cd585806a6486a"}, + {file = "ruff-0.7.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:662a63b4971807623f6f90c1fb664613f67cc182dc4d991471c23c541fee62dd"}, + {file = "ruff-0.7.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:876f5e09eaae3eb76814c1d3b68879891d6fde4824c015d48e7a7da4cf066a3a"}, + {file = "ruff-0.7.4-py3-none-win32.whl", hash = "sha256:75c53f54904be42dd52a548728a5b572344b50d9b2873d13a3f8c5e3b91f5cac"}, + {file = "ruff-0.7.4-py3-none-win_amd64.whl", hash = "sha256:745775c7b39f914238ed1f1b0bebed0b9155a17cd8bc0b08d3c87e4703b990d6"}, + {file = "ruff-0.7.4-py3-none-win_arm64.whl", hash = "sha256:11bff065102c3ae9d3ea4dc9ecdfe5a5171349cdd0787c1fc64761212fc9cf1f"}, + {file = "ruff-0.7.4.tar.gz", hash = "sha256:cd12e35031f5af6b9b93715d8c4f40360070b2041f81273d0527683d5708fce2"}, ] [[package]] @@ -1755,6 +1819,17 @@ files = [ docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0,<8.2)", "pytest-mock", "pytest-mypy-testing"] +[[package]] +name = "types-pyyaml" +version = "6.0.12.20240917" +description = "Typing stubs for PyYAML" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-PyYAML-6.0.12.20240917.tar.gz", hash = "sha256:d1405a86f9576682234ef83bcb4e6fff7c9305c8b1fbad5e0bcd4f7dbdc9c587"}, + {file = "types_PyYAML-6.0.12.20240917-py3-none-any.whl", hash = "sha256:392b267f1c0fe6022952462bf5d6523f31e37f6cea49b14cee7ad634b6301570"}, +] + [[package]] name = "typing-extensions" version = "4.12.2" @@ -1794,4 +1869,4 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "4b4047833e23272a9b1182cc2f12f8aab89ff10d027d41d7042c0d84d83cc393" +content-hash = "31610424afe2af7431f13fab806261d5721a01dd2bde239dab13cdeabb4f6596" diff --git a/pyproject.toml b/pyproject.toml index bf21c64..36de0f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,13 +8,18 @@ packages = [ { include = "vessel" } ] +[tool.poetry.group.extra_dependencies] +optional = true + +[tool.poetry.group.qa] +optional = true + [tool.poetry.dependencies] python = "^3.11" diffoscope = "^259" click = "^8.1.7" platformdirs = "^4.2.0" -ruff = "^0.3.1" -portion = "^2.4.2" +portion = "^2.6.0" PyYAML = "^6.0.1" [tool.poetry.group.extra_dependencies.dependencies] @@ -30,6 +35,11 @@ python-debian = "0.1.49" rpm = "0.2.0" python-tlsh = "4.5.0" +[tool.poetry.group.qa.dependencies] +ruff = "^0.7.4" +mypy = "^1.13.0" +types-PyYAML = "^6.0.12" + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" diff --git a/ruff.toml b/ruff.toml index f44f2fc..227654a 100644 --- a/ruff.toml +++ b/ruff.toml @@ -2,7 +2,7 @@ target-version = "py310" line-length = 79 [lint] -select = ["ALL"] +select = ["E", "F"] ignore = [ "TD002", "TD003", From 3673d4383be5e90dce584d355179b40c07a817b5 Mon Sep 17 00:00:00 2001 From: Sebastian Echeverria Date: Fri, 22 Nov 2024 16:31:45 -0500 Subject: [PATCH 02/14] Added actionlint to check for errors in workflow file --- .github/workflows/ci.yaml | 2 +- Makefile | 6 ++++-- poetry.lock | 16 +++++++++++++++- pyproject.toml | 1 + 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6409456..e36ece9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,4 +1,4 @@ -name: Continuous Integration +name: Vessel Linting, Tests and Docker Image Creation on: - push diff --git a/Makefile b/Makefile index 32af44e..2519f99 100644 --- a/Makefile +++ b/Makefile @@ -13,14 +13,16 @@ format: check-format: poetry run ruff format --check -# Lint all source code +# Lint all source code and workflows .PHONY: lint lint: poetry run ruff check --fix + poetry run actionlint .PHONY: check-lint check-lint: poetry run ruff check + poetry run actionlint # Typecheck all source code .PHONY: typecheck @@ -41,7 +43,7 @@ qa: format lint typecheck # Check all QA tasks .PHONY: check -check: check-lint check-typecheck +check: check-format check-lint check-typecheck # Run unit tests with pytest .PHONY: test diff --git a/poetry.lock b/poetry.lock index 9ab6977..ffc3b9a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,19 @@ # This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +[[package]] +name = "actionlint-py" +version = "1.7.4.18" +description = "Python wrapper around invoking actionlint (https://github.com/rhysd/actionlint)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "actionlint_py-1.7.4.18.tar.gz", hash = "sha256:1d54dda00283543304eae93fe2f738a528f223f6067ca37f8b2f025edf06dd42"}, +] + +[package.extras] +auto-update = ["lxml[html-clean]", "requests", "requests_html", "semver"] +dev = ["black"] + [[package]] name = "alembic" version = "1.14.0" @@ -1869,4 +1883,4 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "31610424afe2af7431f13fab806261d5721a01dd2bde239dab13cdeabb4f6596" +content-hash = "775f14982a8ebb35d53757778229aeda596a889ccb706592451859a6042fe1c8" diff --git a/pyproject.toml b/pyproject.toml index 36de0f8..b65cd76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ python-tlsh = "4.5.0" ruff = "^0.7.4" mypy = "^1.13.0" types-PyYAML = "^6.0.12" +actionlint-py = "^1.7.4.18" [build-system] requires = ["poetry-core"] From 4d0168a342866f9497fc3272a72a5d0ced33aa29 Mon Sep 17 00:00:00 2001 From: Sebastian Echeverria Date: Fri, 22 Nov 2024 16:33:16 -0500 Subject: [PATCH 03/14] Restructured readme by setup/running, instead of non-docker/docker --- README.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index f99a7d4..f44347d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ Vessel is a project with the goal of promoting reproducible container builds. The first version of the Vessel tool has one command, `diff`, that compares two built container images and reports on differences between them, flagging as many known issues as possible. The goal of this command is to allow the detection of reproducibility issues when building container images, so that developers can take the appropriate measures to increase reproducibility. -## Dependencies +## Setup + +### Local Environment Setup Pre-requisites: * Linux OS - tested on Ubuntu 22.04 @@ -24,10 +26,17 @@ To set up additional external tools that are used: * Run `diffoscope --list-tools` for a full list. Also, the Dockerfile should install all of them. -Note that it is much simpler to run Vessel in a Docker container, which already contains all these dependencies. See [Docker](#docker). +Note that it is much simpler to run Vessel in a Docker container, which already contains all these dependencies. See [Docker Setup](#docker). + +### Docker Setup + +Assuming you have Docker installed, run the following to build the vessel docker image. + +* `docker build -t vessel .` ## Running +### In Local Environment The tool can be run locally like this: 1. Make sure the environment is active: `poetry shell` @@ -36,14 +45,6 @@ The tool can be run locally like this: Run `vessel --help` for full list of commands and options. -## Docker - -### Building the Docker image - -Assuming you have Docker installed, run: - -* `docker build -t vessel .` - ### Running the Docker container * Note: Running within Docker avoids permission issues during the unpacking of the images From 4656bf2bdf7dd2aba16882ef15352d1dfaafe00b Mon Sep 17 00:00:00 2001 From: Sebastian Echeverria Date: Fri, 22 Nov 2024 16:51:58 -0500 Subject: [PATCH 04/14] Updating readme to use of make, adding pytest, empty for now --- README.md | 19 +++++++++------ poetry.lock | 48 +++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + test/__init__.py | 0 vessel/cli.py | 7 +++--- vessel/utils/diffoscope.py | 3 ++- 6 files changed, 66 insertions(+), 12 deletions(-) create mode 100644 test/__init__.py diff --git a/README.md b/README.md index f44347d..4b33580 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ To set up additional external tools that are used: * Run `diffoscope --list-tools` for a full list. Also, the Dockerfile should install all of them. -Note that it is much simpler to run Vessel in a Docker container, which already contains all these dependencies. See [Docker Setup](#docker). +Note that it is much simpler to run Vessel in a Docker container, which already contains all these dependencies. See [Docker Setup](#docker-setup). ### Docker Setup @@ -65,14 +65,19 @@ Example running on two images from a private Docker registry: ## Development -To lint the code, execute: -* `ruff check` +Follow the instructions at [Local Environment Setup](#local-environment-setup) first to set up your local environment. -To apply the safe lint fixes, execute: -* `ruff check --fix` +To lint the code, and check for format and type issues, execute: +* `make check` -To format the code, execute: -* `ruff format` +To apply the safe lint fixes, and format fixes, execute: +* `make qa` + +To run unit tests, execute: +* `make test` + +To run all the above, execute: +* `make ci` ### Building diff --git a/poetry.lock b/poetry.lock index ffc3b9a..3aa8b1e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -529,6 +529,17 @@ files = [ docs = ["Sphinx", "furo"] test = ["objgraph", "psutil"] +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "ipython" version = "8.29.0" @@ -1407,6 +1418,21 @@ docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-a test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"] type = ["mypy (>=1.11.2)"] +[[package]] +name = "pluggy" +version = "1.5.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + [[package]] name = "portion" version = "2.6.0" @@ -1538,6 +1564,26 @@ docs = ["myst_parser", "sphinx", "sphinx_rtd_theme"] full = ["Pillow (>=8.0.0)", "PyCryptodome", "cryptography"] image = ["Pillow (>=8.0.0)"] +[[package]] +name = "pytest" +version = "7.4.4" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, + {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1883,4 +1929,4 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "775f14982a8ebb35d53757778229aeda596a889ccb706592451859a6042fe1c8" +content-hash = "4421746b635668569027efc5dcfda13e561a92ff71af6ccbbdc227359b2b101f" diff --git a/pyproject.toml b/pyproject.toml index b65cd76..34e6c4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ ruff = "^0.7.4" mypy = "^1.13.0" types-PyYAML = "^6.0.12" actionlint-py = "^1.7.4.18" +pytest = "^7.3.1" [build-system] requires = ["poetry-core"] diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vessel/cli.py b/vessel/cli.py index 95bef27..edb1260 100644 --- a/vessel/cli.py +++ b/vessel/cli.py @@ -79,8 +79,8 @@ def vessel(logging_level: str) -> None: resolve_path=True, ), help=( - "Specify a data directory for unpacking the images. Default: Create a temporary directory that is " - "auto-deleted." + "Specify a data directory for unpacking the images. Default: Create a " + "temporary directory that is auto-deleted." ), ) @click.option( @@ -93,7 +93,8 @@ def vessel(logging_level: str) -> None: resolve_path=True, ), help=( - "Specify a output directory for diffoscope output. Default: Stores in current directory." + "Specify a output directory for diffoscope output. Default: Stores " + "in current directory." ), ) def diff( diff --git a/vessel/utils/diffoscope.py b/vessel/utils/diffoscope.py index b2f5d44..33b9eb0 100644 --- a/vessel/utils/diffoscope.py +++ b/vessel/utils/diffoscope.py @@ -186,7 +186,8 @@ def parse_diffoscope_output( ( diff.comments != [] and not any( - flag.regex["comment"].search(comment) for comment in diff.comments + flag.regex["comment"].search(comment) + for comment in diff.comments ) ) or ( From 808444a57a374e5c0329090d6e7484621fa88402 Mon Sep 17 00:00:00 2001 From: Sebastian Echeverria Date: Fri, 22 Nov 2024 16:59:30 -0500 Subject: [PATCH 05/14] CI: added missing deps --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e36ece9..ba45eaf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,7 +5,7 @@ on: - pull_request jobs: - test: + lint_and_test: runs-on: ubuntu-latest steps: - name: Check out code @@ -17,7 +17,7 @@ jobs: - name: Install poetry run: curl -sSL https://install.python-poetry.org | python - - name: Install dependencies - run: poetry install --with extra_dependencies + run: poetry install --with qa,extra_dependencies - name: Check format run: make check-format - name: Lint code From a175374fd3d289cf7fa414e82dfe21e90bf3f015 Mon Sep 17 00:00:00 2001 From: Sebastian Echeverria Date: Fri, 13 Dec 2024 12:44:18 -0500 Subject: [PATCH 06/14] CI: added explicit import sort step, changed CI make command to only check, not fix --- .github/workflows/ci.yaml | 6 ++++-- Makefile | 17 +++++++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ba45eaf..cb5dc18 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,12 +18,14 @@ jobs: run: curl -sSL https://install.python-poetry.org | python - - name: Install dependencies run: poetry install --with qa,extra_dependencies + - name: Check import sorting + run: make check-isort - name: Check format - run: make check-format + run: make check-format - name: Lint code run: make check-lint - name: Check types - run: make typecheck + run: make check-typecheck - name: Execute unit tests run: make test diff --git a/Makefile b/Makefile index 2519f99..afd11cd 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,15 @@ # QA # ----------------------------------------------------------------------------- +# Sort imports. +.PHONY: isort +isort: + poetry run ruff check --select I --fix + +.PHONY: check-isort +check-isort: + poetry run ruff check --select I + # Format all source code .PHONY: format format: @@ -39,11 +48,11 @@ clean: # All quality assurance .PHONY: qa -qa: format lint typecheck +qa: isort format lint typecheck # Check all QA tasks .PHONY: check -check: check-format check-lint check-typecheck +check: check-isort check-format check-lint check-typecheck # Run unit tests with pytest .PHONY: test @@ -51,7 +60,7 @@ test: poetry run pytest test # ----------------------------------------------------------------------------- -# All actions and checks needed to update and review for pushing. +# All actions and checks equivalent to what the CI does. # ----------------------------------------------------------------------------- .PHONY: ci -ci: clean qa test +ci: clean check test From 334688b690eb92b66c6ec4076136011c564e0b26 Mon Sep 17 00:00:00 2001 From: Sebastian Echeverria Date: Fri, 13 Dec 2024 13:13:35 -0500 Subject: [PATCH 07/14] Flag: flag attributes were being abused, first as strings, then as regex patterns. Split into separate things. --- vessel/diff/diff_command.py | 4 ++-- vessel/utils/flag.py | 17 +++++++++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/vessel/diff/diff_command.py b/vessel/diff/diff_command.py index 9b8d593..4f0c212 100644 --- a/vessel/diff/diff_command.py +++ b/vessel/diff/diff_command.py @@ -155,10 +155,10 @@ def _setup(self: "DiffCommand") -> bool: flag["comment"], flag["indiff"], ) - for key in temp_flag.regex: + for key in temp_flag.regex_str: try: temp_flag.regex[key] = re.compile( - temp_flag.regex[key], + temp_flag.regex_str[key], ) except re.error: logger.exception( diff --git a/vessel/utils/flag.py b/vessel/utils/flag.py index 420a5c9..bad2b0a 100644 --- a/vessel/utils/flag.py +++ b/vessel/utils/flag.py @@ -25,6 +25,8 @@ """Utility class for flags.""" +from re import Pattern + class Flag: """Class to hold represent a flag for a known issue.""" @@ -42,9 +44,12 @@ def __init__( """Initializer for Flag class.""" self.flag_id = flag_id self.description = description - self.regex = {} - self.regex["filepath"] = filepath - self.regex["filetype"] = filetype - self.regex["command"] = command - self.regex["comment"] = comment - self.regex["indiff"] = indiff + + self.regex_str: dict[str, str] = {} + self.regex_str["filepath"] = filepath + self.regex_str["filetype"] = filetype + self.regex_str["command"] = command + self.regex_str["comment"] = comment + self.regex_str["indiff"] = indiff + + self.regex: dict[str, Pattern] = {} From 7438a3188e20eea6cc73bb174ad5c41d0be84f0b Mon Sep 17 00:00:00 2001 From: Sebastian Echeverria Date: Fri, 13 Dec 2024 13:17:39 -0500 Subject: [PATCH 08/14] Adding missing casts and explicit types --- vessel/diff/diff_command.py | 2 +- vessel/utils/diffoscope.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vessel/diff/diff_command.py b/vessel/diff/diff_command.py index 4f0c212..dddc3ad 100644 --- a/vessel/diff/diff_command.py +++ b/vessel/diff/diff_command.py @@ -68,7 +68,7 @@ def __init__( self.output_dir: str = output_dir self.temp_dir: tempfile.TemporaryDirectory[str] | None = None self.image_uris: list[ImageURI] = [] - self.unpacked_image_paths = [] + self.unpacked_image_paths: list[str] = [] self.umoci_image_paths = [] self.diffoscope_output_file_name = "diffoscope_output.json" self.summary_output_file_name = "summary.json" diff --git a/vessel/utils/diffoscope.py b/vessel/utils/diffoscope.py index 33b9eb0..4ffae63 100644 --- a/vessel/utils/diffoscope.py +++ b/vessel/utils/diffoscope.py @@ -137,7 +137,7 @@ def parse_diffoscope_output( diff.plus_aligned_lines, strict=False, ): - is_binary = current_detail.get("has_internal_linenos") + is_binary = bool(current_detail.get("has_internal_linenos")) for flag in flags: flag_matches = True file_type_1 = "" From d19bfcd288c5981ac41d6422aa42212f69863bbd Mon Sep 17 00:00:00 2001 From: Sebastian Echeverria Date: Fri, 13 Dec 2024 13:18:07 -0500 Subject: [PATCH 09/14] Ruff: start by leaving default rules, which are closest to default black/flake8 --- ruff.toml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/ruff.toml b/ruff.toml index 227654a..80b14b0 100644 --- a/ruff.toml +++ b/ruff.toml @@ -2,13 +2,7 @@ target-version = "py310" line-length = 79 [lint] -select = ["E", "F"] -ignore = [ - "TD002", - "TD003", - "COM812", - "ISC001" -] +select = ["E4", "E7", "E9", "F"] [lint.pydocstyle] convention = "google" From 63ee93c03833194c7d80e9104b4fd9920b3bc6fa Mon Sep 17 00:00:00 2001 From: Sebastian Echeverria Date: Fri, 13 Dec 2024 13:20:10 -0500 Subject: [PATCH 10/14] Added more missing explicit type hints --- vessel/diff/diff_command.py | 2 +- vessel/utils/unified_diff.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vessel/diff/diff_command.py b/vessel/diff/diff_command.py index dddc3ad..ac1d43a 100644 --- a/vessel/diff/diff_command.py +++ b/vessel/diff/diff_command.py @@ -69,7 +69,7 @@ def __init__( self.temp_dir: tempfile.TemporaryDirectory[str] | None = None self.image_uris: list[ImageURI] = [] self.unpacked_image_paths: list[str] = [] - self.umoci_image_paths = [] + self.umoci_image_paths: list[str] = [] self.diffoscope_output_file_name = "diffoscope_output.json" self.summary_output_file_name = "summary.json" self.unified_diff_output_file_name = "unified_diffs.json" diff --git a/vessel/utils/unified_diff.py b/vessel/utils/unified_diff.py index a2e40cd..21eb09a 100644 --- a/vessel/utils/unified_diff.py +++ b/vessel/utils/unified_diff.py @@ -245,8 +245,8 @@ def issues_from_difflines( :return: List of flagged issues, list of unknown issues, updated intervals in each line that haven't been matched by regex """ - flagged_issues = [] - unknown_issues = [] + flagged_issues: list[dict[str, Any]] = [] + unknown_issues: list[dict[str, Any]] = [] minus_matched_intervals = ( [ match.span() @@ -347,7 +347,7 @@ def make_issue_dict( minus_str: str | None, plus_str: str | None, flag: Flag | None = None, -) -> dict: +) -> dict[str, Any]: """Create issue dict object. Used to ensure consistency in all issue objects that From 3ce255c1e38cae1c704f83a3fef4aa9b28fe363a Mon Sep 17 00:00:00 2001 From: Sebastian Echeverria Date: Fri, 13 Dec 2024 13:26:24 -0500 Subject: [PATCH 11/14] Added type ignore for portion, since it does not have type hints --- vessel/utils/unified_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vessel/utils/unified_diff.py b/vessel/utils/unified_diff.py index 21eb09a..1093f04 100644 --- a/vessel/utils/unified_diff.py +++ b/vessel/utils/unified_diff.py @@ -29,7 +29,7 @@ from logging import getLogger from typing import Any -import portion +import portion # type: ignore from vessel.utils.flag import Flag From ebcbe15973120e50ec801d7c17adc27c44290822 Mon Sep 17 00:00:00 2001 From: Sebastian Echeverria Date: Fri, 13 Dec 2024 13:31:07 -0500 Subject: [PATCH 12/14] Updated readme with proper make command instructions --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4b33580..5c1be57 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ To apply the safe lint fixes, and format fixes, execute: To run unit tests, execute: * `make test` -To run all the above, execute: +To run all checks and tests in a clean environment, similar to the Ci workflow, execute: * `make ci` ### Building From d38dd352a150a4f554583a734132da12bb4f0c78 Mon Sep 17 00:00:00 2001 From: Sebastian Echeverria Date: Fri, 13 Dec 2024 13:36:35 -0500 Subject: [PATCH 13/14] Readme: added instructions for additional deps --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5c1be57..5053249 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Pre-requisites: To set up the Python environment and the required packages: 1. `poetry shell` -2. `poetry install` +2. `poetry install --with extra_dependencies` To set up additional external tools that are used: * Install the skopeo package (e.g., `apt-get install skopeo`) @@ -67,6 +67,9 @@ Example running on two images from a private Docker registry: Follow the instructions at [Local Environment Setup](#local-environment-setup) first to set up your local environment. +To install the dev dependencies, run: +* `poetry install --with qa` + To lint the code, and check for format and type issues, execute: * `make check` From b0607b41721e62613faa655afd29d217a76883bb Mon Sep 17 00:00:00 2001 From: Sebastian Echeverria Date: Fri, 13 Dec 2024 13:36:51 -0500 Subject: [PATCH 14/14] CI: disabling test running for now, since there are no tests and it fails --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cb5dc18..15b3585 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -26,8 +26,8 @@ jobs: run: make check-lint - name: Check types run: make check-typecheck - - name: Execute unit tests - run: make test + # - name: Execute unit tests + # run: make test docker: runs-on: ubuntu-latest