circuitpython_build_tools/build.py

@@ -24,34 +24,44 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+import functools
+import multiprocessing
 import os
 import os.path
 import platform
 import pathlib
+import re
 import requests
 import semver
 import shutil
 import stat
 import sys
 import subprocess
 import tempfile
+import platformdirs
+
+@functools.cache
+def _git_version():                                                 
+    version_str = subprocess.check_output(["git", "--version"], encoding="ascii", errors="replace")                    
+    version_str = re.search("([0-9]\.*)*[0-9]", version_str).group(0)
+    return tuple(int(part) for part in version_str.split("."))
+
+def git_filter_arg():
+    clone_supports_filter = (                    
+        False if "NO_USE_CLONE_FILTER" in os.environ else _git_version() >= (2, 36, 0)
+    )                                                                            
+
+    if clone_supports_filter:                                                
+        return ["--filter=blob:none"]
+    else:                                                              
+        return []
 
 # pyproject.toml `py_modules` values that are incorrect. These should all have PRs filed!
 # and should be removed when the fixed version is incorporated in its respective bundle.
 
 pyproject_py_modules_blocklist = set((
-    # adafruit bundle
-    "adafruit_colorsys",
-
     # community bundle
     "at24mac_eeprom",
-    "circuitpython_Candlesticks",
-    "CircuitPython_Color_Picker",
-    "CircuitPython_Equalizer",
-    "CircuitPython_Scales",
-    "circuitPython_Slider",
-    "circuitpython_uboxplot",
-    "P1AM",
     "p1am_200_helpers",
 ))
 
@@ -60,6 +70,8 @@
 else:
     from tomli import loads as load_toml
 
+mpy_cross_path = platformdirs.user_cache_path("circuitpython-build-tools", ensure_exists=True)
+
 def load_pyproject_toml(lib_path: pathlib.Path):
     try:
         return load_toml((lib_path / "pyproject.toml") .read_text(encoding="utf-8"))
@@ -106,9 +118,14 @@ def version_string(path=None, *, valid_semver=False):
             version = commitish
     return version
 
-def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False):
+def mpy_cross(version, quiet=False):
+    circuitpython_tag = version["tag"]
+    name = version["name"]
+    ext = ".exe" * (os.name == "nt")
+    mpy_cross_filename = mpy_cross_path / f"mpy-cross-{name}{ext}"
+
     if os.path.isfile(mpy_cross_filename):
-        return
+        return mpy_cross_filename
 
     # Try to pull from S3
     uname = platform.uname()
@@ -136,7 +153,7 @@ def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False):
                     os.chmod(mpy_cross_filename, os.stat(mpy_cross_filename)[0] | stat.S_IXUSR)
                     if not quiet:
                         print("  FOUND")
-                    return
+                    return mpy_cross_filename
         except Exception as e:
             if not quiet:
                 print(f"    exception fetching from S3: {e}")
@@ -149,26 +166,21 @@ def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False):
         print(title)
         print("=" * len(title))
 
-    os.makedirs("build_deps/", exist_ok=True)
-    if not os.path.isdir("build_deps/circuitpython"):
-        clone = subprocess.run("git clone https://github.com/adafruit/circuitpython.git build_deps/circuitpython", shell=True)
-        if clone.returncode != 0:
-            sys.exit(clone.returncode)
+    build_dir = mpy_cross_path / f"build-circuitpython-{circuitpython_tag}"
+    if not os.path.isdir(build_dir):
+        subprocess.check_call(["git", "clone", *git_filter_arg(), "-b", circuitpython_tag, "https://github.com/adafruit/circuitpython.git", build_dir])
 
-    current_dir = os.getcwd()
-    os.chdir("build_deps/circuitpython")
-    make = subprocess.run("git fetch && git checkout {TAG} && git submodule update".format(TAG=circuitpython_tag), shell=True)
-    os.chdir("tools")
-    make = subprocess.run("git submodule update --init .", shell=True)
-    os.chdir("../mpy-cross")
-    make = subprocess.run("make clean && make", shell=True)
-    os.chdir(current_dir)
+    subprocess.check_call(["git", "submodule", "update", "--recursive"], cwd=build_dir)
+    subprocess.check_call([sys.executable, "tools/ci_fetch_deps.py", "mpy-cross"], cwd=build_dir)
+    subprocess.check_call(["make", "clean"], cwd=build_dir / "mpy-cross")
+    subprocess.check_call(["make", f"-j{multiprocessing.cpu_count()}"], cwd=build_dir / "mpy-cross")
 
-    if make.returncode != 0:
-        print("Failed to build mpy-cross from source... bailing out")
-        sys.exit(make.returncode)
+    mpy_built = build_dir / f"mpy-cross/build/mpy-cross{ext}"
+    if not os.path.exists(mpy_built):
+        mpy_built = build_dir / f"mpy-cross/mpy-cross{ext}"
 
-    shutil.copy("build_deps/circuitpython/mpy-cross/mpy-cross", mpy_cross_filename)
+    shutil.copy(mpy_built, mpy_cross_filename)
+    return mpy_cross_filename
 
 def _munge_to_temp(original_path, temp_file, library_version):
     with open(original_path, "r", encoding="utf-8") as original_file:
@@ -276,12 +288,6 @@ def library(library_path, output_directory, package_folder_prefix,
     example_files = package_info["example_files"]
     module_name = package_info["module_name"]
 
-    for fn in example_files:
-        base_dir = os.path.join(output_directory.replace("/lib", "/"),
-                                fn.relative_to(library_path).parent)
-        if not os.path.isdir(base_dir):
-            os.makedirs(base_dir)
-
     for fn in py_package_files:
         base_dir = os.path.join(output_directory,
                                 fn.relative_to(library_path).parent)
@@ -311,7 +317,7 @@ def library(library_path, output_directory, package_folder_prefix,
                             if mpy_success != 0:
                                 raise RuntimeError("mpy-cross failed on", full_path)
                         else:
-                            shutil.copyfile(full_path, output_file)
+                            shutil.copyfile(temp_file_name, output_file)
                     finally:
                         os.remove(temp_file_name)
             else:
@@ -338,6 +344,12 @@ def library(library_path, output_directory, package_folder_prefix,
 
     for filename in example_files:
         full_path = os.path.join(library_path, filename)
+
+        relative_filename_parts = list(filename.relative_to(library_path).parts)
+        relative_filename_parts.insert(1, library_path.split(os.path.sep)[-1])
+        final_relative_filename = os.path.join(*relative_filename_parts)
         output_file = os.path.join(output_directory.replace("/lib", "/"),
-                                   filename.relative_to(library_path))
+                                   final_relative_filename)
+
+        os.makedirs(os.path.join(*output_file.split(os.path.sep)[:-1]), exist_ok=True)
         shutil.copyfile(full_path, output_file)

circuitpython_build_tools/scripts/build_bundles.py

@@ -41,7 +41,6 @@
 else:
     import importlib.metadata as importlib_metadata
 
-
 BLINKA_LIBRARIES = [
     "adafruit-blinka",
     "adafruit-blinka-bleio",
@@ -280,10 +279,8 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d
 
     # Build .mpy bundle(s)
     if "mpy" not in ignore:
-        os.makedirs("build_deps", exist_ok=True)
         for version in target_versions.VERSIONS:
-            mpy_cross = "build_deps/mpy-cross-" + version["name"] + (".exe" * (os.name == "nt"))
-            build.mpy_cross(mpy_cross, version["tag"])
+            mpy_cross = build.mpy_cross(version)
             zip_filename = os.path.join(output_directory,
                 filename_prefix + '-{TAG}-mpy-{VERSION}.zip'.format(
                     TAG=version["name"],

circuitpython_build_tools/scripts/build_mpy_cross.py

@@ -28,9 +28,14 @@
 import os
 import sys
 
+import click
+
+@click.command
+@click.argument("versions")
+def main(versions):
+    print(versions)
+    for version in [v for v in target_versions.VERSIONS if v['name'] in versions]:
+        print(f"{version['name']}: {build.mpy_cross(version)}")
+
 if __name__ == "__main__":
-    output_directory = sys.argv[1]
-    os.makedirs(output_directory, exist_ok=True)
-    for version in target_versions.VERSIONS:
-        mpy_cross = output_directory + "/mpy-cross-" + version["name"]
-        build.mpy_cross(mpy_cross, version["tag"])
+    main()

circuitpython_build_tools/scripts/circuitpython_mpy_cross.py

@@ -0,0 +1,21 @@
+import subprocess
+
+import click
+
+from ..target_versions import VERSIONS
+from ..build import mpy_cross
+
+@click.command(context_settings={"ignore_unknown_options": True})
+@click.option("--circuitpython-version", type=click.Choice([version["name"] for version in VERSIONS]))
+@click.option("--quiet/--no-quiet", "quiet", type=bool, default=True)
+@click.argument("mpy-cross-args", nargs=-1, required=True)
+def main(circuitpython_version, quiet, mpy_cross_args):
+    version_info, = [v for v in VERSIONS if v["name"] == circuitpython_version]
+    mpy_cross_exe = str(mpy_cross(version_info, quiet))
+    try:
+        subprocess.check_call([mpy_cross_exe, *mpy_cross_args])
+    except subprocess.CalledProcessError as e:
+        raise SystemExit(e.returncode)
+
+if __name__ == '__main__':
+    main()

circuitpython_build_tools/target_versions.py

@@ -25,6 +25,5 @@
 # The tag specifies which version of CircuitPython to use for mpy-cross.
 # The name is used when constructing the zip file names.
 VERSIONS = [
-    {"tag": "8.2.0", "name": "8.x"},
-    {"tag": "9.0.0-alpha.2", "name": "9.x"},
+    {"tag": "9.2.4", "name": "9.x"},
 ]

requirements.txt

@@ -3,3 +3,4 @@ requests
 semver
 wheel
 tomli; python_version < "3.11"
+platformdirs

setup.py

@@ -14,9 +14,10 @@
       package_data={'circuitpython_build_tools': ['data/mpy-cross-*']},
       zip_safe=False,
       python_requires='>=3.10',
-      install_requires=['Click', 'requests', 'semver', 'tomli; python_version < "3.11"'],
+      install_requires=['Click', 'requests', 'semver', 'tomli; python_version < "3.11"', 'platformdirs'],
       entry_points='''
         [console_scripts]
         circuitpython-build-bundles=circuitpython_build_tools.scripts.build_bundles:build_bundles
+        circuitpython-mpy-cross=circuitpython_build_tools.scripts.circuitpython_mpy_cross:main
       '''
       )