diff --git a/.github/workflows/python-build-package.yml b/.github/workflows/python-build-package.yml index 2b3f5109..e9d0fe3b 100644 --- a/.github/workflows/python-build-package.yml +++ b/.github/workflows/python-build-package.yml @@ -16,7 +16,7 @@ permissions: contents: read jobs: - build: + build-wheels: runs-on: ${{ matrix.platform.runner }} strategy: matrix: @@ -34,7 +34,7 @@ jobs: with: python-version: '3.12' - name: Install uv - run: curl -LsSf https://astral.sh/uv/0.4.7/install.sh | sh + run: curl -LsSf https://astral.sh/uv/0.5.22/install.sh | sh - if: matrix.platform.runner == 'ubuntu-latest' uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: @@ -76,14 +76,7 @@ jobs: - run: "python3 -c 'import magika; m = magika.Magika(); print(m)'" - run: magika -r tests_data/basic - run: python3 ./python/scripts/run_quick_test_magika_cli.py - # Windows' onnxruntime/ort returns different results wrt. Linux/MacOS. For - # now keep building even in case of misdetections. - # TODO(https://github.com/google/magika/issues/892): remove when this is fixed. - continue-on-error: ${{ matrix.platform.runner == 'windows-latest' }} - run: python3 ./python/scripts/run_quick_test_magika_module.py - # TODO(https://github.com/google/magika/issues/892): same as above, - # remove when this is fixed. - continue-on-error: ${{ matrix.platform.runner == 'windows-latest' }} - name: Upload wheels if: github.event_name != 'pull_request' uses: actions/upload-artifact@v4 @@ -91,20 +84,29 @@ jobs: name: wheels-${{ matrix.platform.runner }}-${{ matrix.platform.target }} path: dist - sdist: + build-pure-python-wheel: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: python3 ./python/scripts/fix_package_version.py - - name: Build sdist - uses: PyO3/maturin-action@v1 - with: - command: sdist - args: --out=../dist - working-directory: python - - name: Upload sdist + - name: Install uv + run: curl -LsSf https://astral.sh/uv/0.5.22/install.sh | sh + - run: uv run ./scripts/prepare_pyproject_for_pure_python_wheel.py + working-directory: python + - name: Build pure python wheel + run: uv build --wheel --out-dir ../dist + working-directory: python + name: Check that `uv add magika.whl` works + run: mkdir /tmp/test-uv && cp -vR dist/*.whl /tmp/test-uv && cd /tmp/test-uv && uv init && uv add ./$(\ls -1 *.whl | head -n 1) + - name: Install wheels + run: python3 -m pip install $(python -c "import glob; print(glob.glob('dist/*.whl')[0])") + - run: magika --version + - run: "python3 -c 'import magika; m = magika.Magika(); print(m)'" + - run: magika -r tests_data/basic + - run: python3 ./python/scripts/run_quick_test_magika_cli.py + - run: python3 ./python/scripts/run_quick_test_magika_module.py + - name: Upload wheel if: github.event_name != 'pull_request' uses: actions/upload-artifact@v4 with: - name: wheels-sdist + name: wheels-pure path: dist diff --git a/.github/workflows/python-test-suite.yml b/.github/workflows/python-test-suite.yml index 9c2d5900..4d4f8805 100644 --- a/.github/workflows/python-test-suite.yml +++ b/.github/workflows/python-test-suite.yml @@ -34,7 +34,7 @@ jobs: python-version: '${{ matrix.python-version }}' - name: Install uv - run: curl -LsSf https://astral.sh/uv/0.4.7/install.sh | sh + run: curl -LsSf https://astral.sh/uv/0.5.22/install.sh | sh - name: Install all projects dependencies (with the requested python version) working-directory: python diff --git a/python/pyproject.toml b/python/pyproject.toml index 394ba185..9f06dbcb 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -2,10 +2,10 @@ name = "magika" description = "A tool to determine the content type of a file with deep learning" authors = [ - {author = "Magika Developers", email = "magika-dev@google.com"}, + {name = "Magika Developers", email = "magika-dev@google.com"}, ] readme = "README.md" -license = "Apache-2.0" +license = {"text" = "Apache-2.0"} requires-python = ">=3.8" keywords = ["content type detection", "machine learning"] classifiers = [ @@ -57,6 +57,8 @@ dev-dependencies = [ "pytest>=8.3.2", "ruff>=0.6.3", "twine>=5.1.1", + "tomli-w>=1.0.0", + "tomli>=2.0.1", ] [build-system] diff --git a/python/scripts/prepare_pyproject_for_pure_python_wheel.py b/python/scripts/prepare_pyproject_for_pure_python_wheel.py new file mode 100644 index 00000000..6b7d9c95 --- /dev/null +++ b/python/scripts/prepare_pyproject_for_pure_python_wheel.py @@ -0,0 +1,45 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from pathlib import Path + +import tomli +import tomli_w + + +def main() -> None: + pyproject_toml_path = Path(__file__).parent.parent / "pyproject.toml" + + pyproject_content = tomli.loads(pyproject_toml_path.read_text()) + + # Remove entry about maturin, we don't need it + _ = pyproject_content["tool"].pop("maturin") + + # Tell uv we want to use the hatchling build system + pyproject_content["build-system"] = { + "requires": ["hatchling"], + "build-backend": "hatchling.build", + } + + # Make the python's magika client available as a script + pyproject_content["project"]["scripts"] = { + "magika": "magika.cli.magika_client:main", + } + + pyproject_toml_path.write_text(tomli_w.dumps(pyproject_content)) + + +if __name__ == "__main__": + main() diff --git a/python/scripts/run_quick_test_magika_cli.py b/python/scripts/run_quick_test_magika_cli.py index 7a9d7293..6a02764a 100755 --- a/python/scripts/run_quick_test_magika_cli.py +++ b/python/scripts/run_quick_test_magika_cli.py @@ -26,19 +26,36 @@ import subprocess import sys from pathlib import Path +from typing import Optional import click @click.command() -def main() -> None: +@click.option( + "--client-path", + type=click.Path(exists=True, file_okay=True, dir_okay=False, resolve_path=True), +) +def main(client_path: Optional[Path]) -> None: + """Tests the Rust or Python Magika client. By default, it runs "magika" + (expected in PATH). Use --client-path to specify a different client + executable. + """ + basic_tests_dir = ( Path(__file__).resolve().parent.parent.parent / "tests_data" / "basic" ) assert basic_tests_dir.is_dir() + if client_path is None: + client_path = Path("magika") + print(f'Testing client: "{client_path}"') + + p = subprocess.run([str(client_path), "--version"], capture_output=True, text=True) + print(f'Output of "magika --version": {p.stdout.strip()}') + p = subprocess.run( - ["magika", "-r", "--label", str(basic_tests_dir)], + [str(client_path), "-r", "--label", "--no-colors", str(basic_tests_dir)], capture_output=True, text=True, ) diff --git a/python/scripts/magika_python_module_tester.py b/python/src/magika/cli/magika_client.py similarity index 98% rename from python/scripts/magika_python_module_tester.py rename to python/src/magika/cli/magika_client.py index a5d2a4ff..6a5658ae 100755 --- a/python/scripts/magika_python_module_tester.py +++ b/python/src/magika/cli/magika_client.py @@ -154,9 +154,6 @@ def main( with_colors = False _l = get_logger(use_colors=with_colors) - _l.warning( - "This CLI is deprecated and only used for testing the python module! Use the Rust CLI instead." - ) if verbose: _l.setLevel(logging.INFO) @@ -164,6 +161,7 @@ def main( _l.setLevel(logging.DEBUG) if output_version: + _l.raw_print_to_stdout("Magika python client") _l.raw_print_to_stdout(f"Magika version: {VERSION}") _l.raw_print_to_stdout(f"Default model: {Magika._get_default_model_name()}") sys.exit(0) diff --git a/python/tests/test_python_module_tester.py b/python/tests/test_python_magika_client.py similarity index 76% rename from python/tests/test_python_module_tester.py rename to python/tests/test_python_magika_client.py index cffeaafa..4baa1c3c 100644 --- a/python/tests/test_python_module_tester.py +++ b/python/tests/test_python_magika_client.py @@ -16,16 +16,16 @@ from pathlib import Path -def test_python_module_tester() -> None: +def test_python_magika_client() -> None: python_root_dir = Path(__file__).parent.parent - python_module_tester_path = ( - python_root_dir / "scripts" / "magika_python_module_tester.py" + python_magika_client_path = ( + python_root_dir / "src" / "magika" / "cli" / "magika_client.py" ).resolve() # quick test to check there are no obvious problems - cmd = [str(python_module_tester_path), "--help"] + cmd = [str(python_magika_client_path), "--help"] subprocess.run(cmd, capture_output=True, check=True) # quick test to check there are no crashes - cmd = [str(python_module_tester_path), str(python_module_tester_path)] + cmd = [str(python_magika_client_path), str(python_magika_client_path)] subprocess.run(cmd, capture_output=True, check=True) diff --git a/python/uv.lock b/python/uv.lock index ef221b37..b6773994 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -199,7 +199,7 @@ name = "click" version = "8.1.7" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "platform_system == 'Windows'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/96/d3/f04c7bfcf5c1862a2a5b845c6b2b360488cf47af55dfa79c98f6a6bf98b5/click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de", size = 336121 } wheels = [ @@ -510,7 +510,6 @@ wheels = [ [[package]] name = "magika" -version = "0.1.0rc2.dev0" source = { editable = "." } dependencies = [ { name = "click" }, @@ -529,6 +528,8 @@ dev = [ { name = "mypy" }, { name = "pytest" }, { name = "ruff" }, + { name = "tomli" }, + { name = "tomli-w" }, { name = "twine" }, ] @@ -548,6 +549,8 @@ dev = [ { name = "mypy", specifier = ">=1.11.2" }, { name = "pytest", specifier = ">=8.3.2" }, { name = "ruff", specifier = ">=0.6.3" }, + { name = "tomli", specifier = ">=2.0.1" }, + { name = "tomli-w", specifier = ">=1.0.0" }, { name = "twine", specifier = ">=5.1.1" }, ] @@ -1186,6 +1189,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", size = 12757 }, ] +[[package]] +name = "tomli-w" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/05/6bf21838623186b91aedbda06248ad18f03487dc56fbc20e4db384abde6c/tomli_w-1.0.0.tar.gz", hash = "sha256:f463434305e0336248cac9c2dc8076b707d8a12d019dd349f5c1e382dd1ae1b9", size = 6531 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/01/1da9c66ecb20f31ed5aa5316a957e0b1a5e786a0d9689616ece4ceaf1321/tomli_w-1.0.0-py3-none-any.whl", hash = "sha256:9f2a07e8be30a0729e533ec968016807069991ae2fd921a78d42f429ae5f4463", size = 5984 }, +] + [[package]] name = "traitlets" version = "5.14.3"