Improve sdist generation for Python packages

They no longer require torch/featomic/metatensor dependencies
since these are only required to build the wheels. They also
automatically package the Rust/C++ sources in a tarball

.github/workflows/build-wheels.yml

@@ -59,7 +59,7 @@ jobs:
         run: docker build -t rustc-manylinux2014_x86_64 python/scripts/rustc-manylinux2014_x86_64
 
       - name: build featomic wheel
-        run: python -m cibuildwheel .
+        run: python -m cibuildwheel python/featomic
         env:
           CIBW_BUILD: cp310-*
           CIBW_SKIP: "*musllinux*"
@@ -97,7 +97,7 @@ jobs:
       - name: build featomic sdist
         run: |
           pip install build
-          python -m build --sdist .
+          python -m build --sdist python/featomic
 
       - name: create C++ tarballs
         run: |

.github/workflows/coverage.yml

@@ -72,9 +72,7 @@ jobs:
 
       - name: combine Python coverage files
         run: |
-            coverage combine --append \
-              ./.coverage \
-              ./python/featomic-torch/.coverage
+            coverage combine --append ./python/featomic/.coverage ./python/featomic-torch/.coverage
             coverage xml
 
       - name: upload to codecov.io

.github/workflows/python-tests.yml

@@ -90,7 +90,7 @@ jobs:
           python -m pip install tox
 
       - name: python build tests
-        run: tox -e build-python
+        run: tox -e build-tests
         env:
           # Use the CPU only version of torch when building/running the code
           PIP_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu

python/featomic-torch/MANIFEST.in

@@ -1,10 +1,7 @@
-global-exclude *.pyc
-global-exclude .DS_Store
+include featomic-torch-cxx-*.tar.gz
+include git_extra_version
+include build-backend/backend.py
 
 include pyproject.toml
 include AUTHORS
 include LICENSE
-
-include featomic-torch.tar.gz
-
-include build-backend/backend.py

python/featomic-torch/build-backend/backend.py

@@ -7,9 +7,17 @@
 from setuptools import build_meta
 
 
-ROOT = os.path.realpath(os.path.dirname(__file__))
-FEATOMIC_SRC = os.path.realpath(os.path.join(ROOT, "..", "..", ".."))
-if os.path.exists(os.path.join(FEATOMIC_SRC, "featomic")):
+ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__), ".."))
+FEATOMIC_SRC = os.path.realpath(os.path.join(ROOT, "..", "featomic"))
+FORCED_FEATOMIC_VERSION = os.environ.get("FEATOMIC_TORCH_BUILD_WITH_FEATOMIC_VERSION")
+
+
+if FORCED_FEATOMIC_VERSION is not None:
+    # force a specific version for metatensor-core, this is used when checking the build
+    # from a sdist on a non-released version
+    FEATOMIC_DEP = f"featomic =={FORCED_FEATOMIC_VERSION}"
+
+elif os.path.exists(FEATOMIC_SRC):
     # we are building from a git checkout
 
     # add a random uuid to the file url to prevent pip from using a cached
@@ -21,6 +29,7 @@
     FEATOMIC_DEP = "featomic >=0.1.0.dev0,<0.2.0"
 
 
+get_requires_for_build_sdist = build_meta.get_requires_for_build_sdist
 prepare_metadata_for_build_wheel = build_meta.prepare_metadata_for_build_wheel
 build_wheel = build_meta.build_wheel
 build_sdist = build_meta.build_sdist
@@ -33,8 +42,3 @@ def get_requires_for_build_wheel(config_settings=None):
         "metatensor-torch >=0.6.0,<0.7.0",
         FEATOMIC_DEP,
     ]
-
-
-def get_requires_for_build_sdist(config_settings=None):
-    defaults = build_meta.get_requires_for_build_sdist(config_settings)
-    return defaults + [FEATOMIC_DEP]

python/featomic-torch/setup.py

@@ -12,28 +12,8 @@
 
 ROOT = os.path.realpath(os.path.dirname(__file__))
 
-FEATOMIC_SRC = os.path.join(ROOT, "..", "..", "featomic")
-
-FEATOMIC_TORCH_SRC = os.path.join(ROOT, "..", "..", "featomic-torch")
-if not os.path.exists(FEATOMIC_TORCH_SRC):
-    # we are building from a sdist, which should include featomic-torch
-    # sources as a tarball
-    tarballs = glob.glob(os.path.join(ROOT, "featomic-torch-cxx-*.tar.gz"))
-
-    if not len(tarballs) == 1:
-        raise RuntimeError(
-            "expected a single 'featomic-torch-cxx-*.tar.gz' file containing "
-            "featomic-torch C++ sources"
-        )
-
-    FEATOMIC_TORCH_SRC = os.path.realpath(tarballs[0])
-    subprocess.run(
-        ["cmake", "-E", "tar", "xf", FEATOMIC_TORCH_SRC],
-        cwd=ROOT,
-        check=True,
-    )
-
-    FEATOMIC_TORCH_SRC = ".".join(FEATOMIC_TORCH_SRC.split(".")[:-2])
+FEATOMIC_PYTHON_SRC = os.path.realpath(os.path.join(ROOT, "..", "featomic"))
+FEATOMIC_TORCH_SRC = os.path.realpath(os.path.join(ROOT, "..", "..", "featomic-torch"))
 
 
 class cmake_ext(build_ext):
@@ -149,20 +129,54 @@ def run(self):
         )
 
 
-class sdist_git_version(sdist):
+class sdist_generate_data(sdist):
     """
-    Create a sdist with an additional generated file containing the extra
-    version from git.
+    Create a sdist with an additional generated files:
+        - `git_extra_version`
+        - `featomic-torch-cxx-*.tar.gz`
     """
 
     def run(self):
         with open("git_extra_version", "w") as fd:
             fd.write(git_extra_version())
 
+        generate_cxx_tar()
+
         # run original sdist
         super().run()
 
         os.unlink("git_extra_version")
+        for path in glob.glob("featomic-torch-cxx-*.tar.gz"):
+            os.unlink(path)
+
+
+def generate_cxx_tar():
+    script = os.path.join(ROOT, "..", "..", "scripts", "package-featomic-torch.sh")
+    assert os.path.exists(script)
+
+    try:
+        output = subprocess.run(
+            ["bash", "--version"],
+            stderr=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            encoding="utf8",
+        )
+    except Exception as e:
+        raise RuntimeError("could not run `bash`, is it installed?") from e
+
+    output = subprocess.run(
+        ["bash", script, os.getcwd()],
+        stderr=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        encoding="utf8",
+    )
+    if output.returncode != 0:
+        stderr = output.stderr
+        stdout = output.stdout
+        raise RuntimeError(
+            "failed to collect C++ sources for Python sdist\n"
+            f"stdout:\n {stdout}\n\nstderr:\n {stderr}"
+        )
 
 
 def git_extra_version():
@@ -223,6 +237,26 @@ def git_extra_version():
 
 
 if __name__ == "__main__":
+    if not os.path.exists(FEATOMIC_TORCH_SRC):
+        # we are building from a sdist, which should include featomic-torch
+        # sources as a tarball
+        tarballs = glob.glob(os.path.join(ROOT, "featomic-torch-cxx-*.tar.gz"))
+
+        if not len(tarballs) == 1:
+            raise RuntimeError(
+                "expected a single 'featomic-torch-cxx-*.tar.gz' file containing "
+                "featomic-torch C++ sources"
+            )
+
+        FEATOMIC_TORCH_SRC = os.path.realpath(tarballs[0])
+        subprocess.run(
+            ["cmake", "-E", "tar", "xf", FEATOMIC_TORCH_SRC],
+            cwd=ROOT,
+            check=True,
+        )
+
+        FEATOMIC_TORCH_SRC = ".".join(FEATOMIC_TORCH_SRC.split(".")[:-2])
+
     if os.path.exists("git_extra_version"):
         # we are building from a sdist, without git available, but the git
         # version was recorded in a git_extra_version file
@@ -247,14 +281,13 @@ def git_extra_version():
         "torch >= 1.12",
         "metatensor-torch >=0.6.0,<0.7.0",
     ]
-    if os.path.exists(FEATOMIC_SRC):
+    if os.path.exists(FEATOMIC_PYTHON_SRC):
         # we are building from a git checkout
-        featomic_path = os.path.realpath(os.path.join(ROOT, "..", ".."))
 
         # add a random uuid to the file url to prevent pip from using a cached
         # wheel for featomic, and force it to re-build from scratch
         uuid = uuid.uuid4()
-        install_requires.append(f"featomic @ file://{featomic_path}?{uuid}")
+        install_requires.append(f"featomic @ file://{FEATOMIC_PYTHON_SRC}?{uuid}")
     else:
         # we are building from a sdist/installing from a wheel
         install_requires.append("featomic >=0.1.0.dev0,<0.2.0")
@@ -269,7 +302,7 @@ def git_extra_version():
         cmdclass={
             "build_ext": cmake_ext,
             "bdist_egg": bdist_egg if "bdist_egg" in sys.argv else bdist_egg_disabled,
-            "sdist": sdist_git_version,
+            "sdist": sdist_generate_data,
         },
         package_data={
             "featomic-torch": [

python/featomic/AUTHORS

@@ -0,0 +1 @@
+../../AUTHORS

python/featomic/LICENSE

@@ -0,0 +1 @@
+../../LICENSE

python/featomic/MANIFEST.in

@@ -0,0 +1,6 @@
+include featomic-cxx-*.tar.gz
+include git_extra_version
+
+include pyproject.toml
+include AUTHORS
+include LICENSE

python/featomic/README.rst

@@ -0,0 +1 @@
+../../README.rst

pyproject.toml → python/featomic/pyproject.toml

@@ -51,27 +51,11 @@ build-backend = "setuptools.build_meta"
 zip-safe = true
 
 [tool.setuptools.packages.find]
-where = ["python/featomic"]
 include = ["featomic*"]
 namespaces = false
 
 ### ======================================================================== ###
 
-
-[tool.ruff.lint]
-select = ["E", "F", "B", "I"]
-ignore = ["B018", "B904"]
-
-[tool.ruff.lint.isort]
-lines-after-imports = 2
-known-first-party = ["featomic"]
-known-third-party = ["torch"]
-
-[tool.ruff.format]
-docstring-code-format = true
-
-### ======================================================================== ###
-
 [tool.pytest.ini_options]
 python_files = ["*.py"]
-testpaths = ["python/featomic/tests"]
+testpaths = ["tests"]