diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index e5cf07676..bad68e588 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -59,7 +59,7 @@ jobs: run: docker build -t rustc-manylinux2014_x86_64 python/scripts/rustc-manylinux2014_x86_64 - name: build featomic wheel - run: python -m cibuildwheel . + run: python -m cibuildwheel python/featomic env: CIBW_BUILD: cp310-* CIBW_SKIP: "*musllinux*" @@ -97,7 +97,7 @@ jobs: - name: build featomic sdist run: | pip install build - python -m build --sdist . + python -m build --sdist python/featomic - name: create C++ tarballs run: | diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index b185ec9d3..135bee9ac 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -72,9 +72,7 @@ jobs: - name: combine Python coverage files run: | - coverage combine --append \ - ./.coverage \ - ./python/featomic-torch/.coverage + coverage combine --append ./python/featomic/.coverage ./python/featomic-torch/.coverage coverage xml - name: upload to codecov.io diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index f1930b4cb..dd259ae0d 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -90,7 +90,7 @@ jobs: python -m pip install tox - name: python build tests - run: tox -e build-python + run: tox -e build-tests env: # Use the CPU only version of torch when building/running the code PIP_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index c62e5dcf0..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,27 +0,0 @@ -global-exclude *.pyc -global-exclude .DS_Store - -prune docs - -recursive-include featomic * -recursive-include docs/featomic-json-schema * - -# include the minimal crates from the Cargo workspace -include python/Cargo.toml -include python/lib.rs -include featomic-torch/Cargo.toml -include featomic-torch/lib.rs - -include Cargo.* -include pyproject.toml -include AUTHORS -include LICENSE - -prune python/tests -prune python/*.egg-info - -prune featomic/tests -prune featomic/benches/data -prune featomic/examples/data - -exclude tox.ini diff --git a/python/featomic-torch/.gitignore b/python/featomic-torch/.gitignore deleted file mode 100644 index b3bfb1f9c..000000000 --- a/python/featomic-torch/.gitignore +++ /dev/null @@ -1 +0,0 @@ -featomic-torch-*.tar.gz diff --git a/python/featomic-torch/MANIFEST.in b/python/featomic-torch/MANIFEST.in index 64e2d4049..6e946646b 100644 --- a/python/featomic-torch/MANIFEST.in +++ b/python/featomic-torch/MANIFEST.in @@ -1,10 +1,7 @@ -global-exclude *.pyc -global-exclude .DS_Store +include featomic-torch-cxx-*.tar.gz +include git_extra_version +include build-backend/backend.py include pyproject.toml include AUTHORS include LICENSE - -include featomic-torch.tar.gz - -include build-backend/backend.py diff --git a/python/featomic-torch/build-backend/backend.py b/python/featomic-torch/build-backend/backend.py index 42a8267ae..14f3219bf 100644 --- a/python/featomic-torch/build-backend/backend.py +++ b/python/featomic-torch/build-backend/backend.py @@ -7,9 +7,17 @@ from setuptools import build_meta -ROOT = os.path.realpath(os.path.dirname(__file__)) -FEATOMIC_SRC = os.path.realpath(os.path.join(ROOT, "..", "..", "..")) -if os.path.exists(os.path.join(FEATOMIC_SRC, "featomic")): +ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__), "..")) +FEATOMIC_SRC = os.path.realpath(os.path.join(ROOT, "..", "featomic")) +FORCED_FEATOMIC_VERSION = os.environ.get("FEATOMIC_TORCH_BUILD_WITH_FEATOMIC_VERSION") + + +if FORCED_FEATOMIC_VERSION is not None: + # force a specific version for metatensor-core, this is used when checking the build + # from a sdist on a non-released version + FEATOMIC_DEP = f"featomic =={FORCED_FEATOMIC_VERSION}" + +elif os.path.exists(FEATOMIC_SRC): # we are building from a git checkout # add a random uuid to the file url to prevent pip from using a cached @@ -21,6 +29,7 @@ FEATOMIC_DEP = "featomic >=0.1.0.dev0,<0.2.0" +get_requires_for_build_sdist = build_meta.get_requires_for_build_sdist prepare_metadata_for_build_wheel = build_meta.prepare_metadata_for_build_wheel build_wheel = build_meta.build_wheel build_sdist = build_meta.build_sdist @@ -33,8 +42,3 @@ def get_requires_for_build_wheel(config_settings=None): "metatensor-torch >=0.6.0,<0.7.0", FEATOMIC_DEP, ] - - -def get_requires_for_build_sdist(config_settings=None): - defaults = build_meta.get_requires_for_build_sdist(config_settings) - return defaults + [FEATOMIC_DEP] diff --git a/python/featomic-torch/setup.py b/python/featomic-torch/setup.py index 1c5ea2c40..0ba5f4749 100644 --- a/python/featomic-torch/setup.py +++ b/python/featomic-torch/setup.py @@ -12,28 +12,8 @@ ROOT = os.path.realpath(os.path.dirname(__file__)) -FEATOMIC_SRC = os.path.join(ROOT, "..", "..", "featomic") - -FEATOMIC_TORCH_SRC = os.path.join(ROOT, "..", "..", "featomic-torch") -if not os.path.exists(FEATOMIC_TORCH_SRC): - # we are building from a sdist, which should include featomic-torch - # sources as a tarball - tarballs = glob.glob(os.path.join(ROOT, "featomic-torch-cxx-*.tar.gz")) - - if not len(tarballs) == 1: - raise RuntimeError( - "expected a single 'featomic-torch-cxx-*.tar.gz' file containing " - "featomic-torch C++ sources" - ) - - FEATOMIC_TORCH_SRC = os.path.realpath(tarballs[0]) - subprocess.run( - ["cmake", "-E", "tar", "xf", FEATOMIC_TORCH_SRC], - cwd=ROOT, - check=True, - ) - - FEATOMIC_TORCH_SRC = ".".join(FEATOMIC_TORCH_SRC.split(".")[:-2]) +FEATOMIC_PYTHON_SRC = os.path.realpath(os.path.join(ROOT, "..", "featomic")) +FEATOMIC_TORCH_SRC = os.path.realpath(os.path.join(ROOT, "..", "..", "featomic-torch")) class cmake_ext(build_ext): @@ -149,20 +129,54 @@ def run(self): ) -class sdist_git_version(sdist): +class sdist_generate_data(sdist): """ - Create a sdist with an additional generated file containing the extra - version from git. + Create a sdist with an additional generated files: + - `git_extra_version` + - `featomic-torch-cxx-*.tar.gz` """ def run(self): with open("git_extra_version", "w") as fd: fd.write(git_extra_version()) + generate_cxx_tar() + # run original sdist super().run() os.unlink("git_extra_version") + for path in glob.glob("featomic-torch-cxx-*.tar.gz"): + os.unlink(path) + + +def generate_cxx_tar(): + script = os.path.join(ROOT, "..", "..", "scripts", "package-featomic-torch.sh") + assert os.path.exists(script) + + try: + output = subprocess.run( + ["bash", "--version"], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + encoding="utf8", + ) + except Exception as e: + raise RuntimeError("could not run `bash`, is it installed?") from e + + output = subprocess.run( + ["bash", script, os.getcwd()], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + encoding="utf8", + ) + if output.returncode != 0: + stderr = output.stderr + stdout = output.stdout + raise RuntimeError( + "failed to collect C++ sources for Python sdist\n" + f"stdout:\n {stdout}\n\nstderr:\n {stderr}" + ) def git_extra_version(): @@ -223,6 +237,26 @@ def git_extra_version(): if __name__ == "__main__": + if not os.path.exists(FEATOMIC_TORCH_SRC): + # we are building from a sdist, which should include featomic-torch + # sources as a tarball + tarballs = glob.glob(os.path.join(ROOT, "featomic-torch-cxx-*.tar.gz")) + + if not len(tarballs) == 1: + raise RuntimeError( + "expected a single 'featomic-torch-cxx-*.tar.gz' file containing " + "featomic-torch C++ sources" + ) + + FEATOMIC_TORCH_SRC = os.path.realpath(tarballs[0]) + subprocess.run( + ["cmake", "-E", "tar", "xf", FEATOMIC_TORCH_SRC], + cwd=ROOT, + check=True, + ) + + FEATOMIC_TORCH_SRC = ".".join(FEATOMIC_TORCH_SRC.split(".")[:-2]) + if os.path.exists("git_extra_version"): # we are building from a sdist, without git available, but the git # version was recorded in a git_extra_version file @@ -247,14 +281,13 @@ def git_extra_version(): "torch >= 1.12", "metatensor-torch >=0.6.0,<0.7.0", ] - if os.path.exists(FEATOMIC_SRC): + if os.path.exists(FEATOMIC_PYTHON_SRC): # we are building from a git checkout - featomic_path = os.path.realpath(os.path.join(ROOT, "..", "..")) # add a random uuid to the file url to prevent pip from using a cached # wheel for featomic, and force it to re-build from scratch uuid = uuid.uuid4() - install_requires.append(f"featomic @ file://{featomic_path}?{uuid}") + install_requires.append(f"featomic @ file://{FEATOMIC_PYTHON_SRC}?{uuid}") else: # we are building from a sdist/installing from a wheel install_requires.append("featomic >=0.1.0.dev0,<0.2.0") @@ -269,7 +302,7 @@ def git_extra_version(): cmdclass={ "build_ext": cmake_ext, "bdist_egg": bdist_egg if "bdist_egg" in sys.argv else bdist_egg_disabled, - "sdist": sdist_git_version, + "sdist": sdist_generate_data, }, package_data={ "featomic-torch": [ diff --git a/python/featomic/AUTHORS b/python/featomic/AUTHORS new file mode 120000 index 000000000..f04b7e8a2 --- /dev/null +++ b/python/featomic/AUTHORS @@ -0,0 +1 @@ +../../AUTHORS \ No newline at end of file diff --git a/python/featomic/LICENSE b/python/featomic/LICENSE new file mode 120000 index 000000000..30cff7403 --- /dev/null +++ b/python/featomic/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/featomic/MANIFEST.in b/python/featomic/MANIFEST.in new file mode 100644 index 000000000..294112b15 --- /dev/null +++ b/python/featomic/MANIFEST.in @@ -0,0 +1,6 @@ +include featomic-cxx-*.tar.gz +include git_extra_version + +include pyproject.toml +include AUTHORS +include LICENSE diff --git a/python/featomic/README.rst b/python/featomic/README.rst new file mode 120000 index 000000000..c768ff7d9 --- /dev/null +++ b/python/featomic/README.rst @@ -0,0 +1 @@ +../../README.rst \ No newline at end of file diff --git a/pyproject.toml b/python/featomic/pyproject.toml similarity index 82% rename from pyproject.toml rename to python/featomic/pyproject.toml index 9061e4bb9..59a0c25db 100644 --- a/pyproject.toml +++ b/python/featomic/pyproject.toml @@ -51,27 +51,11 @@ build-backend = "setuptools.build_meta" zip-safe = true [tool.setuptools.packages.find] -where = ["python/featomic"] include = ["featomic*"] namespaces = false ### ======================================================================== ### - -[tool.ruff.lint] -select = ["E", "F", "B", "I"] -ignore = ["B018", "B904"] - -[tool.ruff.lint.isort] -lines-after-imports = 2 -known-first-party = ["featomic"] -known-third-party = ["torch"] - -[tool.ruff.format] -docstring-code-format = true - -### ======================================================================== ### - [tool.pytest.ini_options] python_files = ["*.py"] -testpaths = ["python/featomic/tests"] +testpaths = ["tests"] diff --git a/python/featomic/setup.py b/python/featomic/setup.py new file mode 100644 index 000000000..97da2c5cf --- /dev/null +++ b/python/featomic/setup.py @@ -0,0 +1,340 @@ +import glob +import os +import shutil +import subprocess +import sys +import uuid + +from setuptools import Extension, setup +from setuptools.command.bdist_egg import bdist_egg +from setuptools.command.build_ext import build_ext +from setuptools.command.sdist import sdist +from wheel.bdist_wheel import bdist_wheel + + +ROOT = os.path.realpath(os.path.dirname(__file__)) +FEATOMIC_SRC = os.path.realpath(os.path.join(ROOT, "..", "..", "featomic")) + +FEATOMIC_BUILD_TYPE = os.environ.get("FEATOMIC_BUILD_TYPE", "release") +if FEATOMIC_BUILD_TYPE not in ["debug", "release"]: + raise Exception( + f"invalid build type passed: '{FEATOMIC_BUILD_TYPE}'," + "expected 'debug' or 'release'" + ) + + +FEATOMIC_TORCH_SRC = os.path.join(ROOT, "..", "featomic-torch") + + +class universal_wheel(bdist_wheel): + """Helper class for override wheel tag. + + When building the wheel, the `wheel` package assumes that if we have a + binary extension then we are linking to `libpython.so`; and thus the wheel + is only usable with a single python version. This is not the case for + here, and the wheel will be compatible with any Python >=3.6. This is + tracked in https://github.com/pypa/wheel/issues/185, but until then we + manually override the wheel tag. + """ + + def get_tag(self): + """Get the tag for override.""" + tag = bdist_wheel.get_tag(self) + # tag[2:] contains the os/arch tags, we want to keep them + return ("py3", "none") + tag[2:] + + +class cmake_ext(build_ext): + """Build the native library using cmake.""" + + def run(self): + """Run cmake build and install the resulting library.""" + source_dir = FEATOMIC_SRC + build_dir = os.path.join(ROOT, "build", "cmake-build") + install_dir = os.path.join(os.path.realpath(self.build_lib), "featomic") + + try: + os.mkdir(build_dir) + except OSError: + pass + + cmake_options = [ + f"-DCMAKE_INSTALL_PREFIX={install_dir}", + "-DCMAKE_INSTALL_LIBDIR=lib", + f"-DCMAKE_BUILD_TYPE={FEATOMIC_BUILD_TYPE}", + "-DFEATOMIC_FETCH_METATENSOR=ON", + "-DFEATOMIC_INSTALL_BOTH_STATIC_SHARED=OFF", + "-DBUILD_SHARED_LIBS=ON", + "-DEXTRA_RUST_FLAGS=-Cstrip=symbols", + ] + + if "CARGO" in os.environ: + cmake_options.append(f"-DCARGO_EXE={os.environ['CARGO']}") + + # Handle cross-compilation by detecting cibuildwheels environnement + # variables + if sys.platform.startswith("darwin"): + # ARCHFLAGS is set by cibuildwheels + ARCHFLAGS = os.environ.get("ARCHFLAGS") + if ARCHFLAGS is not None: + archs = filter( + lambda u: bool(u), + ARCHFLAGS.strip().split("-arch "), + ) + archs = list(archs) + assert len(archs) == 1 + arch = archs[0].strip() + + if arch == "x86_64": + cmake_options.append("-DRUST_BUILD_TARGET=x86_64-apple-darwin") + elif arch == "arm64": + cmake_options.append("-DRUST_BUILD_TARGET=aarch64-apple-darwin") + else: + raise ValueError(f"unknown arch: {arch}") + + elif sys.platform.startswith("linux"): + # we set RUST_BUILD_TARGET in our custom docker image + RUST_BUILD_TARGET = os.environ.get("RUST_BUILD_TARGET") + if RUST_BUILD_TARGET is not None: + cmake_options.append(f"-DRUST_BUILD_TARGET={RUST_BUILD_TARGET}") + + elif sys.platform.startswith("win32"): + # CARGO_BUILD_TARGET is set by cibuildwheels + CARGO_BUILD_TARGET = os.environ.get("CARGO_BUILD_TARGET") + if CARGO_BUILD_TARGET is not None: + cmake_options.append(f"-DRUST_BUILD_TARGET={CARGO_BUILD_TARGET}") + + else: + raise ValueError(f"unknown platform: {sys.platform}") + + subprocess.run( + ["cmake", source_dir, *cmake_options], + cwd=build_dir, + check=True, + ) + subprocess.run( + ["cmake", "--build", build_dir, "--parallel", "--target", "install"], + check=True, + ) + + # do not include metatensor libraries/headers/cmake config within + # featomic wheel + for file in glob.glob(os.path.join(install_dir, "lib", "libmetatensor.*")): + os.unlink(file) + + for file in glob.glob(os.path.join(install_dir, "bin", "metatensor.dll")): + os.unlink(file) + + shutil.rmtree(os.path.join(install_dir, "lib", "cmake", "metatensor")) + + for file in glob.glob(os.path.join(install_dir, "include", "metatensor*")): + os.unlink(file) + + +class bdist_egg_disabled(bdist_egg): + """Disabled version of bdist_egg + + Prevents setup.py install performing setuptools' default easy_install, + which it should never ever do. + """ + + def run(self): + sys.exit( + "Aborting implicit building of eggs. " + + "Use `pip install .` or `python setup.py bdist_wheel && pip " + + "uninstall metatensor -y && pip install dist/metatensor-*.whl` " + + "to install from source." + ) + + +class sdist_generate_data(sdist): + """ + Create a sdist with an additional generated files: + - `git_extra_version` + - `featomic-cxx-*.tar.gz` + """ + + def run(self): + with open("git_extra_version", "w") as fd: + fd.write(git_extra_version()) + + generate_cxx_tar() + + # run original sdist + super().run() + + os.unlink("git_extra_version") + for path in glob.glob("featomic-cxx-*.tar.gz"): + os.unlink(path) + + +def generate_cxx_tar(): + script = os.path.join(ROOT, "..", "..", "scripts", "package-featomic.sh") + assert os.path.exists(script) + + try: + output = subprocess.run( + ["bash", "--version"], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + encoding="utf8", + ) + except Exception as e: + raise RuntimeError("could not run `bash`, is it installed?") from e + + output = subprocess.run( + ["bash", script, os.getcwd()], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + encoding="utf8", + ) + if output.returncode != 0: + stderr = output.stderr + stdout = output.stdout + raise RuntimeError( + "failed to collect C++ sources for Python sdist\n" + f"stdout:\n {stdout}\n\nstderr:\n {stderr}" + ) + + +def get_rust_version(): + # read version from Cargo.toml + with open(os.path.join(FEATOMIC_SRC, "Cargo.toml")) as fd: + for line in fd: + if line.startswith("version"): + _, version = line.split(" = ") + # remove quotes + version = version[1:-2] + # take the first version in the file, this should be the right + # version + break + + return version + + +def git_extra_version(): + """ + If git is available, it is used to check if we are installing a development + version or a released version (by checking how many commits happened since + the last tag). + """ + + # Add pre-release info the version + try: + tags_list = subprocess.run( + ["git", "tag"], + stderr=subprocess.DEVNULL, + stdout=subprocess.PIPE, + check=True, + ) + tags_list = tags_list.stdout.decode("utf8").strip() + + if tags_list == "": + first_commit = subprocess.run( + ["git", "rev-list", "--max-parents=0", "HEAD"], + stderr=subprocess.DEVNULL, + stdout=subprocess.PIPE, + check=True, + ) + reference = first_commit.stdout.decode("utf8").strip() + + else: + last_tag = subprocess.run( + ["git", "describe", "--tags", "--abbrev=0"], + stderr=subprocess.DEVNULL, + stdout=subprocess.PIPE, + check=True, + ) + + reference = last_tag.stdout.decode("utf8").strip() + + except Exception: + reference = "" + pass + + try: + n_commits_since_tag = subprocess.run( + ["git", "rev-list", f"{reference}..HEAD", "--count"], + stderr=subprocess.DEVNULL, + stdout=subprocess.PIPE, + check=True, + ) + n_commits_since_tag = n_commits_since_tag.stdout.decode("utf8").strip() + + if n_commits_since_tag != 0: + return ".dev" + n_commits_since_tag + except Exception: + pass + + return "" + + +if __name__ == "__main__": + if not os.path.exists(FEATOMIC_SRC): + # we are building from a sdist, which should include featomic Rust + # sources as a tarball + tarballs = glob.glob(os.path.join(ROOT, "featomic-*.tar.gz")) + + if not len(tarballs) == 1: + raise RuntimeError( + "expected a single 'featomic-*.tar.gz' file containing " + "featomic Rust sources. remove all files and re-run " + "scripts/package-featomic.sh" + ) + + FEATOMIC_SRC = os.path.realpath(tarballs[0]) + subprocess.run( + ["cmake", "-E", "tar", "xf", FEATOMIC_SRC], + cwd=ROOT, + check=True, + ) + + FEATOMIC_SRC = ".".join(FEATOMIC_SRC.split(".")[:-2]) + + if os.path.exists("git_extra_version"): + # we are building from a sdist, without git available, but the git + # version was recorded in a git_extra_version file + with open("git_extra_version") as fd: + extra_version = fd.read() + else: + extra_version = git_extra_version() + + version = get_rust_version() + extra_version + + with open(os.path.join(ROOT, "AUTHORS")) as fd: + authors = fd.read().splitlines() + + extras_require = {} + if os.path.exists(FEATOMIC_TORCH_SRC): + # we are building from a git checkout + + # add a random uuid to the file url to prevent pip from using a cached + # wheel for featomic-torch, and force it to re-build from scratch + uuid = uuid.uuid4() + extras_require["torch"] = f"featomic-torch @ file://{FEATOMIC_TORCH_SRC}?{uuid}" + else: + # we are building from a sdist/installing from a wheel + extras_require["torch"] = "featomic-torch >=0.1.0.dev0,<0.2.0" + + setup( + version=version, + author=", ".join(authors), + extras_require=extras_require, + ext_modules=[ + # only declare the extension, it is built & copied as required by cmake + # in the build_ext command + Extension(name="featomic", sources=[]), + ], + cmdclass={ + "build_ext": cmake_ext, + "bdist_egg": bdist_egg if "bdist_egg" in sys.argv else bdist_egg_disabled, + "bdist_wheel": universal_wheel, + "sdist": sdist_generate_data, + }, + package_data={ + "featomic": [ + "featomic/lib/*", + "featomic/include/*", + ] + }, + ) diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 000000000..0c20eca99 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,11 @@ +[lint] +select = ["E", "F", "B", "I"] +ignore = ["B018", "B904"] + +[lint.isort] +lines-after-imports = 2 +known-first-party = ["featomic"] +known-third-party = ["torch"] + +[format] +docstring-code-format = true diff --git a/scripts/build-all-wheels.sh b/scripts/build-all-wheels.sh new file mode 100644 index 000000000..a0fec589f --- /dev/null +++ b/scripts/build-all-wheels.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -eux + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd) +cd "$ROOT_DIR" + +TMP_DIR="$1" +rm -rf "$TMP_DIR"/dist + +# check building sdist from a checkout, and wheel from the sdist +python -m build python/featomic --outdir "$TMP_DIR"/dist + +# get the version of featomic we just built +FEATOMIC_VERSION=$(basename "$(find "$TMP_DIR"/dist -name "featomic-*.tar.gz")" | cut -d - -f 2) +FEATOMIC_VERSION=${FEATOMIC_VERSION%.tar.gz} + +# for featomic-torch, we need a pre-built version of featomic, so +# we use the one we just generated and make it available to pip +dir2pi --no-symlink "$TMP_DIR"/dist + +PORT=8912 +if nc -z localhost $PORT; then + printf "\033[91m ERROR: an application is listening to port %d. Please free up the port first. \033[0m\n" $PORT >&2 + exit 1 +fi + +PYPI_SERVER_PID="" +function cleanup() { + kill $PYPI_SERVER_PID +} +# Make sure to stop the Python server on script exit/cancellation +trap cleanup INT TERM EXIT + +python -m http.server --directory "$TMP_DIR"/dist $PORT & +PYPI_SERVER_PID=$! + +# add the python server to the set of extra pip index URL +export PIP_EXTRA_INDEX_URL="http://localhost:$PORT/simple/ ${PIP_EXTRA_INDEX_URL=}" +# force featomic-torch to use a specific featomic version when building +export FEATOMIC_TORCH_BUILD_WITH_FEATOMIC_VERSION="$FEATOMIC_VERSION" + +# build featomic-torch, using featomic from `PIP_EXTRA_INDEX_URL` +# for the sdist => wheel build. +python -m build python/featomic-torch --outdir "$TMP_DIR/dist" diff --git a/scripts/clean-python.sh b/scripts/clean-python.sh index 8e456c1a8..b23cd475a 100755 --- a/scripts/clean-python.sh +++ b/scripts/clean-python.sh @@ -10,12 +10,18 @@ cd "$ROOT_DIR" rm -rf dist rm -rf build -rm -rf .coverage + rm -rf docs/build rm -rf docs/src/examples +rm -rf python/featomic/dist +rm -rf python/featomic/build +rm -rf python/featomic/featomic-cxx-*.tar.gz + rm -rf python/featomic-torch/dist rm -rf python/featomic-torch/build +rm -rf python/featomic-torch/featomic-torch-cxx-*.tar.gz find . -name "*.egg-info" -exec rm -rf "{}" + find . -name "__pycache__" -exec rm -rf "{}" + +find . -name ".coverage" -exec rm -rf "{}" + diff --git a/setup.py b/setup.py index e99b08903..dafbcfeb0 100644 --- a/setup.py +++ b/setup.py @@ -1,282 +1,24 @@ -import glob +# This is not the actual setup.py for this project, see `python/featomic/setup.py` for +# it. Instead, this file is here to enable `pip install .` from a git checkout or `pip +# install git+https://...` without having to specify a subdirectory + import os -import shutil -import subprocess -import sys -import uuid -from setuptools import Extension, setup -from setuptools.command.bdist_egg import bdist_egg -from setuptools.command.build_ext import build_ext -from setuptools.command.sdist import sdist -from wheel.bdist_wheel import bdist_wheel +from setuptools import setup ROOT = os.path.realpath(os.path.dirname(__file__)) -FEATOMIC_TORCH = os.path.join(ROOT, "python", "featomic-torch") - -FEATOMIC_BUILD_TYPE = os.environ.get("FEATOMIC_BUILD_TYPE", "release") -if FEATOMIC_BUILD_TYPE not in ["debug", "release"]: - raise Exception( - f"invalid build type passed: '{FEATOMIC_BUILD_TYPE}'," - "expected 'debug' or 'release'" - ) - - -class universal_wheel(bdist_wheel): - """Helper class for override wheel tag. - - When building the wheel, the `wheel` package assumes that if we have a - binary extension then we are linking to `libpython.so`; and thus the wheel - is only usable with a single python version. This is not the case for - here, and the wheel will be compatible with any Python >=3.6. This is - tracked in https://github.com/pypa/wheel/issues/185, but until then we - manually override the wheel tag. - """ - - def get_tag(self): - """Get the tag for override.""" - tag = bdist_wheel.get_tag(self) - # tag[2:] contains the os/arch tags, we want to keep them - return ("py3", "none") + tag[2:] - -class cmake_ext(build_ext): - """Build the native library using cmake.""" - - def run(self): - """Run cmake build and install the resulting library.""" - source_dir = os.path.join(ROOT, "featomic") - build_dir = os.path.join(ROOT, "build", "cmake-build") - install_dir = os.path.join(os.path.realpath(self.build_lib), "featomic") - - try: - os.mkdir(build_dir) - except OSError: - pass - - cmake_options = [ - f"-DCMAKE_INSTALL_PREFIX={install_dir}", - "-DCMAKE_INSTALL_LIBDIR=lib", - f"-DCMAKE_BUILD_TYPE={FEATOMIC_BUILD_TYPE}", - "-DFEATOMIC_FETCH_METATENSOR=ON", - "-DFEATOMIC_INSTALL_BOTH_STATIC_SHARED=OFF", - "-DBUILD_SHARED_LIBS=ON", - "-DEXTRA_RUST_FLAGS=-Cstrip=symbols", +setup( + name="featomic-git", + version="0.0.0", + install_requires=[ + f"featomic @ file://{ROOT}/python/featomic", + ], + extras_require={ + "torch": [ + f"featomic-torch @ file://{ROOT}/python/featomic-torch", ] - - if "CARGO" in os.environ: - cmake_options.append(f"-DCARGO_EXE={os.environ['CARGO']}") - - # Handle cross-compilation by detecting cibuildwheels environnement - # variables - if sys.platform.startswith("darwin"): - # ARCHFLAGS is set by cibuildwheels - ARCHFLAGS = os.environ.get("ARCHFLAGS") - if ARCHFLAGS is not None: - archs = filter( - lambda u: bool(u), - ARCHFLAGS.strip().split("-arch "), - ) - archs = list(archs) - assert len(archs) == 1 - arch = archs[0].strip() - - if arch == "x86_64": - cmake_options.append("-DRUST_BUILD_TARGET=x86_64-apple-darwin") - elif arch == "arm64": - cmake_options.append("-DRUST_BUILD_TARGET=aarch64-apple-darwin") - else: - raise ValueError(f"unknown arch: {arch}") - - elif sys.platform.startswith("linux"): - # we set RUST_BUILD_TARGET in our custom docker image - RUST_BUILD_TARGET = os.environ.get("RUST_BUILD_TARGET") - if RUST_BUILD_TARGET is not None: - cmake_options.append(f"-DRUST_BUILD_TARGET={RUST_BUILD_TARGET}") - - elif sys.platform.startswith("win32"): - # CARGO_BUILD_TARGET is set by cibuildwheels - CARGO_BUILD_TARGET = os.environ.get("CARGO_BUILD_TARGET") - if CARGO_BUILD_TARGET is not None: - cmake_options.append(f"-DRUST_BUILD_TARGET={CARGO_BUILD_TARGET}") - - else: - raise ValueError(f"unknown platform: {sys.platform}") - - subprocess.run( - ["cmake", source_dir, *cmake_options], - cwd=build_dir, - check=True, - ) - subprocess.run( - ["cmake", "--build", build_dir, "--parallel", "--target", "install"], - check=True, - ) - - # do not include metatensor libraries/headers/cmake config within - # featomic wheel - for file in glob.glob(os.path.join(install_dir, "lib", "libmetatensor.*")): - os.unlink(file) - - for file in glob.glob(os.path.join(install_dir, "bin", "metatensor.dll")): - os.unlink(file) - - shutil.rmtree(os.path.join(install_dir, "lib", "cmake", "metatensor")) - - for file in glob.glob(os.path.join(install_dir, "include", "metatensor*")): - os.unlink(file) - - -class bdist_egg_disabled(bdist_egg): - """Disabled version of bdist_egg - - Prevents setup.py install performing setuptools' default easy_install, - which it should never ever do. - """ - - def run(self): - sys.exit( - "Aborting implicit building of eggs. " - + "Use `pip install .` or `python setup.py bdist_wheel && pip " - + "uninstall metatensor -y && pip install dist/metatensor-*.whl` " - + "to install from source." - ) - - -class sdist_git_version(sdist): - """ - Create a sdist with an additional generated file containing the extra - version from git. - """ - - def run(self): - with open("git_extra_version", "w") as fd: - fd.write(git_extra_version()) - - # run original sdist - super().run() - - os.unlink("git_extra_version") - - -def get_rust_version(): - # read version from Cargo.toml - with open(os.path.join(ROOT, "featomic", "Cargo.toml")) as fd: - for line in fd: - if line.startswith("version"): - _, version = line.split(" = ") - # remove quotes - version = version[1:-2] - # take the first version in the file, this should be the right - # version - break - - return version - - -def git_extra_version(): - """ - If git is available, it is used to check if we are installing a development - version or a released version (by checking how many commits happened since - the last tag). - """ - - # Add pre-release info the version - try: - tags_list = subprocess.run( - ["git", "tag"], - stderr=subprocess.DEVNULL, - stdout=subprocess.PIPE, - check=True, - ) - tags_list = tags_list.stdout.decode("utf8").strip() - - if tags_list == "": - first_commit = subprocess.run( - ["git", "rev-list", "--max-parents=0", "HEAD"], - stderr=subprocess.DEVNULL, - stdout=subprocess.PIPE, - check=True, - ) - reference = first_commit.stdout.decode("utf8").strip() - - else: - last_tag = subprocess.run( - ["git", "describe", "--tags", "--abbrev=0"], - stderr=subprocess.DEVNULL, - stdout=subprocess.PIPE, - check=True, - ) - - reference = last_tag.stdout.decode("utf8").strip() - - except Exception: - reference = "" - pass - - try: - n_commits_since_tag = subprocess.run( - ["git", "rev-list", f"{reference}..HEAD", "--count"], - stderr=subprocess.DEVNULL, - stdout=subprocess.PIPE, - check=True, - ) - n_commits_since_tag = n_commits_since_tag.stdout.decode("utf8").strip() - - if n_commits_since_tag != 0: - return ".dev" + n_commits_since_tag - except Exception: - pass - - return "" - - -if __name__ == "__main__": - if os.path.exists("git_extra_version"): - # we are building from a sdist, without git available, but the git - # version was recorded in a git_extra_version file - with open("git_extra_version") as fd: - extra_version = fd.read() - else: - extra_version = git_extra_version() - - version = get_rust_version() + extra_version - - with open(os.path.join(ROOT, "AUTHORS")) as fd: - authors = fd.read().splitlines() - - extras_require = {} - if os.path.exists(FEATOMIC_TORCH): - # we are building from a git checkout - - # add a random uuid to the file url to prevent pip from using a cached - # wheel for featomic-torch, and force it to re-build from scratch - uuid = uuid.uuid4() - extras_require["torch"] = f"featomic-torch @ file://{FEATOMIC_TORCH}?{uuid}" - else: - # we are building from a sdist/installing from a wheel - extras_require["torch"] = "featomic-torch >=0.1.0.dev0,<0.2.0" - - setup( - version=version, - author=", ".join(authors), - extras_require=extras_require, - ext_modules=[ - # only declare the extension, it is built & copied as required by cmake - # in the build_ext command - Extension(name="featomic", sources=[]), - ], - cmdclass={ - "build_ext": cmake_ext, - "bdist_egg": bdist_egg if "bdist_egg" in sys.argv else bdist_egg_disabled, - "bdist_wheel": universal_wheel, - "sdist": sdist_git_version, - }, - package_data={ - "featomic": [ - "featomic/lib/*", - "featomic/include/*", - ] - }, - ) + }, + packages=[], +) diff --git a/tox.ini b/tox.ini index ff1d27797..07e5fa3a3 100644 --- a/tox.ini +++ b/tox.ini @@ -57,7 +57,7 @@ deps = {[testenv]packaging_deps} commands = - pip wheel . {[testenv]build-single-wheel} --wheel-dir {envtmpdir}/dist + pip wheel python/featomic {[testenv]build-single-wheel} --wheel-dir {envtmpdir}/dist [testenv:all-deps] @@ -77,6 +77,7 @@ deps = pyscf;platform_system!="Windows" wigners +changedir = python/featomic commands = pytest {[testenv]test_options} {posargs} @@ -87,6 +88,7 @@ deps = pytest pytest-cov +changedir = python/featomic commands = pytest {[testenv]test_options} {posargs} @@ -160,36 +162,27 @@ commands = ruff check --fix-only {[testenv]lint-folders} -[testenv:build-python] +[testenv:build-tests] +description = Asserts Pythons package build integrity so one can build sdist and wheels package = skip -# Make sure we can build sdist and a wheel for python deps = - twine build + twine # a tool to check sdist and wheels metadata + pip2pi # tool to create PyPI-like package indexes + setuptools -allowlist_externals = - bash - +allowlist_externals = bash commands = python --version # print the version of python used in this test - bash ./scripts/package-featomic-torch.sh python/featomic-torch/ - - bash -c "rm -rf {envtmpdir}/dist" - - # check building sdist from a checkout, and wheel from the sdist - python -m build . --outdir {envtmpdir}/dist - - # for featomic-torch, we can not build from a sdist until featomic - # is available on PyPI, so we build both sdist and wheel from a checkout - python -m build python/featomic-torch --sdist --outdir {envtmpdir}/dist - python -m build python/featomic-torch --wheel --outdir {envtmpdir}/dist + bash ./scripts/build-all-wheels.sh {envtmpdir} twine check {envtmpdir}/dist/*.tar.gz twine check {envtmpdir}/dist/*.whl # check building wheels directly from the a checkout - python -m build . --wheel --outdir {envtmpdir}/dist + python -m build python/featomic --wheel --outdir {envtmpdir}/dist + python -m build python/featomic-torch --wheel --outdir {envtmpdir}/dist [flake8]