[WIP]: Updates for Pyodide builds after pyodide-build was unvendored

cibuildwheel/pyodide.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 import os
 import shutil
 import sys
@@ -41,6 +42,7 @@ class PythonConfiguration:
     version: str
     identifier: str
     pyodide_version: str
+    pyodide_build_version: str
     emscripten_version: str
     node_version: str
 
@@ -65,11 +67,63 @@ def install_emscripten(tmp: Path, version: str) -> Path:
     return emcc_path
 
 
-def install_xbuildenv(env: dict[str, str], pyodide_version: str) -> str:
+def search_xbuildenv(env: dict[str, str]) -> list[str]:
+    """Searches for the compatible xbuildenvs for the current pyodide-build version"""
+    with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"):
+        xbuildenvs = call(
+            "pyodide",
+            "xbuildenv",
+            "search",
+            "--json",
+            "--all",
+            env=env,
+            cwd=CIBW_CACHE_PATH,
+            capture_stdout=True,
+        ).strip()
+        xbuildenvs_dict = json.loads(xbuildenvs)
+        compatible_xbuildenvs = [
+            env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"]
+        ]
+        # Fetch just the "stable" versions
+        compatible_xbuildenvs_filtered = [
+            version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc")
+        ]
+        # TODO: possibly remove that? Since this won't allow testing the unstable/dev versions
+
+        return compatible_xbuildenvs_filtered
+
+
+# The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml,
+# which will always be compatible. Hence, this condition really checks only for the case where the
+# version is supplied manually through a CIBW_PYODIDE_VERSION environment variable and raises an
+# error as appropriate.
+def validate_xbuildenv(
+    cibw_pyodide_version: str, pyodide_build_version: str, compatible_versions: list[str]
+) -> None:
+    """Validate the Pyodide version if set manually for the current pyodide-build version
+    against a list of compatible versions."""
+
+    if cibw_pyodide_version not in compatible_versions:
+        msg = (
+            f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build"
+            f" version {pyodide_build_version}. The compatible versions available to download are:"
+            f" {compatible_versions}. Please use the 'pyodide xbuildenv search' command to"
+            f" find the compatible versions for {pyodide_build_version}"
+        )
+        raise errors.FatalError(msg)
+
+
+def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str:
+    """Install a particular Pyodide xbuildenv version and set a path to the Pyodide root."""
+    # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are
+    # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall
+    # specify the pyodide-build version in the root path, which will set up the xbuildenv for
+    # the requested Pyodide version.
     pyodide_root = (
         CIBW_CACHE_PATH
-        / f".pyodide-xbuildenv-{pyodide_version}/{pyodide_version}/xbuildenv/pyodide-root"
+        / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root"
     )
+
     with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"):
         if pyodide_root.exists():
             return str(pyodide_root)
@@ -78,6 +132,8 @@ def install_xbuildenv(env: dict[str, str], pyodide_version: str) -> str:
         # PYODIDE_ROOT so copy it first.
         env = dict(env)
         env.pop("PYODIDE_ROOT", None)
+
+        # 3. Install the xbuildenv
         call(
             "pyodide",
             "xbuildenv",
@@ -165,13 +221,28 @@ def setup_python(
         env=env,
     )
 
-    log.step("Installing emscripten...")
+    log.step(f"Installing Emscripten version: {python_configuration.emscripten_version} ...")
     emcc_path = install_emscripten(tmp, python_configuration.emscripten_version)
 
     env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]])
 
-    log.step("Installing Pyodide xbuildenv...")
-    env["PYODIDE_ROOT"] = install_xbuildenv(env, python_configuration.pyodide_version)
+    # Allow overriding the xbuildenv version with an environment variable. This allows
+    # testing new Pyodide xbuildenv versions before they are officially released, or for
+    # using a different Pyodide version other than the one that is listed to be compatible
+    # with the pyodide-build version in the build-platforms.toml file.
+    cibw_pyodide_version = os.environ.get(
+        "CIBW_PYODIDE_VERSION", python_configuration.pyodide_version
+    )
+    log.step(f"Installing Pyodide xbuildenv version: {cibw_pyodide_version} ...")
+    # Search for compatible xbuildenv versions
+    compatible_versions = search_xbuildenv(env)
+    # and then validate the xbuildenv version
+    validate_xbuildenv(
+        cibw_pyodide_version, python_configuration.pyodide_build_version, compatible_versions
+    )
+    env["PYODIDE_ROOT"] = install_xbuildenv(
+        env, python_configuration.pyodide_build_version, cibw_pyodide_version
+    )
 
     return env
 
@@ -219,7 +290,10 @@ def build(options: Options, tmp_path: Path) -> None:
 
             log.build_start(config.identifier)
 
-            identifier_tmp_dir = tmp_path / config.identifier
+            # Include both the identifier and the Pyodide version in the temp directory name
+            cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", config.pyodide_version)
+            identifier_tmp_dir = tmp_path / f"{config.identifier}_{cibw_pyodide_version}"
+
             built_wheel_dir = identifier_tmp_dir / "built_wheel"
             repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel"
             identifier_tmp_dir.mkdir()

cibuildwheel/resources/build-platforms.toml

@@ -172,5 +172,5 @@ python_configurations = [
 
 [pyodide]
 python_configurations = [
-  { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", emscripten_version = "3.1.58", node_version = "v20" },
+  { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" },
 ]

cibuildwheel/resources/cibuildwheel.schema.json

@@ -814,6 +814,11 @@
         },
         "test-requires": {
           "$ref": "#/properties/test-requires"
+        },
+        "pyodide-version": {
+          "description": "Specify the Pyodide xbuildenv version to use for building",
+          "type": "string",
+          "title": "CIBW_PYODIDE_VERSION"
         }
       }
     }

cibuildwheel/resources/constraints-pyodide312.txt

@@ -19,8 +19,6 @@ charset-normalizer==3.3.2
     # via requests
 click==8.1.7
     # via typer
-cloudpickle==3.0.0
-    # via loky
 cmake==3.30.3
     # via pyodide-build
 distlib==0.3.8
@@ -40,8 +38,6 @@ idna==3.10
     #   requests
 leb128==1.0.8
     # via auditwheel-emscripten
-loky==3.4.1
-    # via pyodide-build
 markdown-it-py==3.0.0
     # via rich
 mdurl==0.1.2
@@ -64,18 +60,16 @@ pydantic-core==2.23.4
     # via pydantic
 pygments==2.18.0
     # via rich
-pyodide-build==0.26.1
+pyodide-build==0.29.0
     # via -r .nox/update_constraints/tmp/constraints-pyodide.in
 pyodide-cli==0.2.4
     # via
     #   auditwheel-emscripten
     #   pyodide-build
-pyodide-lock==0.1.0a6
+pyodide-lock==0.1.0a7
     # via pyodide-build
 pyproject-hooks==1.1.0
     # via build
-pyyaml==6.0.2
-    # via pyodide-build
 requests==2.32.3
     # via pyodide-build
 resolvelib==1.0.1
@@ -100,8 +94,6 @@ typer==0.12.5
     #   auditwheel-emscripten
     #   pyodide-build
     #   pyodide-cli
-types-requests==2.32.0.20240914
-    # via pyodide-build
 typing-extensions==4.12.2
     # via
     #   pydantic
@@ -110,9 +102,7 @@ typing-extensions==4.12.2
 unearth==0.17.2
     # via pyodide-build
 urllib3==2.2.3
-    # via
-    #   requests
-    #   types-requests
+    # via requests
 virtualenv==20.26.5
     # via
     #   build

noxfile.py

@@ -102,10 +102,10 @@ def update_constraints(session: nox.Session) -> None:
     pyodides = build_platforms["pyodide"]["python_configurations"]
     for pyodide in pyodides:
         python_version = ".".join(pyodide["version"].split(".")[:2])
-        pyodide_version = pyodide["pyodide_version"]
+        pyodide_build_version = pyodide["pyodide_build_version"]
         output_file = resources / f"constraints-pyodide{python_version.replace('.', '')}.txt"
         tmp_file = Path(session.create_tmp()) / "constraints-pyodide.in"
-        tmp_file.write_text(f"pip\nbuild[virtualenv]\npyodide-build=={pyodide_version}")
+        tmp_file.write_text(f"pip\nbuild[virtualenv]\npyodide-build=={pyodide_build_version}")
         session.run(
             "uv",
             "pip",

test/test_emscripten.py

@@ -20,7 +20,7 @@
 
 def check_node():
     # cibuildwheel adds a pinned node version to the PATH
-    # check it's in the PATH then, check it's the one that runs pyoodide
+    # check it's in the PATH then, check it's the one that runs pyodide
     cibw_cache_path = Path(sys.argv[1]).resolve(strict=True)
     # find the node executable in PATH
     node = shutil.which("node")