Merge pull request #16 from home-assistant/dev

Release 0.7

azure-pipelines.yml

@@ -88,7 +88,8 @@ jobs:
       sudo apt-get update
       sudo apt-get install -y --no-install-recommends \
         qemu-user-static \
-        binfmt-support
+        binfmt-support \
+        curl
       
       sudo mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc
       sudo update-binfmts --enable qemu-arm
@@ -103,11 +104,13 @@ jobs:
   - script: sudo docker pull homeassistant/$(buildArch)-wheels:$(versionWheels)
     displayName: 'Install wheels builder'
   - script: |
+      curl -s -o requirements_diff.txt https://raw.githubusercontent.com/home-assistant/hassio-wheels/master/requirements.txt
       sudo docker run --rm -v $(pwd):/data:ro -v $(pwd)/.ssh:/root/.ssh:rw \
         homeassistant/$(buildArch)-wheels:$(versionWheels) \
         --apk "build-base;libffi-dev;openssl-dev" \
-        --index https://wheels.hass.io \
+        --index $(wheelsIndex) \
         --requirement requirements.txt \
+        --requirement-diff requirements_diff.txt \
         --upload rsync \
         --remote wheels@$(wheelsHost):/opt/wheels
     displayName: 'Run wheels build'

builder/__main__.py

@@ -10,12 +10,17 @@
 
 from builder.apk import install_apks
 from builder.infra import create_wheels_folder, create_wheels_index
-from builder.pip import build_wheels, extract_packages
+from builder.pip import (
+    build_wheels_package,
+    build_wheels_requirement,
+    write_requirement,
+    extract_packages,
+)
 from builder.upload import run_upload
 from builder.utils import check_url
 
 
-@click.command()
+@click.command("builder")
 @click.option("--apk", default="build-base", help="APKs they are needed to build this.")
 @click.option("--index", required=True, help="Index URL of remote wheels repository.")
 @click.option(
@@ -33,31 +38,43 @@
     type=click_pathlib.Path(exists=True),
     help="Python requirement file to calc the different for selective builds.",
 )
-def builder(apk, index, requirement, upload, remote, requirement_diff):
+@click.option(
+    "--single", default=False, help="Install every package as single requirement."
+)
+def builder(apk, index, requirement, upload, remote, requirement_diff, single):
     """Build wheels precompiled for Home Assistant container."""
     install_apks(apk)
     check_url(index)
 
     exit_code = 0
-    timer = 0
     with TemporaryDirectory() as temp_dir:
         output = Path(temp_dir)
 
         wheels_dir = create_wheels_folder(output)
         wheels_index = create_wheels_index(index)
         packages = extract_packages(requirement, requirement_diff)
 
-        for package in packages:
-            print(f"Process package: {package}", flush=True)
+        if single:
+            timer = 0
+            for package in packages:
+                print(f"Process package: {package}", flush=True)
+                try:
+                    build_wheels_package(package, wheels_index, wheels_dir)
+                except CalledProcessError:
+                    exit_code = 109
+
+                if timer < monotonic():
+                    run_upload(upload, output, remote)
+                    timer = monotonic() + 900
+        else:
+            temp_requirement = Path("/tmp/wheels_requirement.txt")
+            write_requirement(temp_requirement, packages)
+
             try:
-                build_wheels(package, wheels_index, wheels_dir)
+                build_wheels_requirement(temp_requirement, wheels_index, wheels_dir)
             except CalledProcessError:
                 exit_code = 109
 
-            if timer < monotonic():
-                run_upload(upload, output, remote)
-                timer = monotonic() + 900
-
         run_upload(upload, output, remote)
 
     sys.exit(exit_code)

builder/pip.py

@@ -6,7 +6,7 @@
 from typing import List, Optional
 
 
-def build_wheels(package: str, index: str, output: Path) -> None:
+def build_wheels_package(package: str, index: str, output: Path) -> None:
     """Build wheels from a requirements file into output."""
     cpu = os.cpu_count() or 4
 
@@ -26,16 +26,36 @@ def build_wheels(package: str, index: str, output: Path) -> None:
     result.check_returncode()
 
 
+def build_wheels_requirement(requirement: Path, index: str, output: Path) -> None:
+    """Build wheels from a requirements file into output."""
+    cpu = os.cpu_count() or 4
+
+    # Modify speed
+    build_env = os.environ.copy()
+    build_env["MAKEFLAGS"] = f"-j{cpu}"
+
+    result = subprocess.run(
+        f"pip3 wheel --progress-bar ascii --wheel-dir {output} --find-links {index} --requirement {requirement}",
+        shell=True,
+        stdout=sys.stdout,
+        stderr=sys.stderr,
+        env=build_env,
+    )
+
+    # Check result of program
+    result.check_returncode()
+
+
 def parse_requirements(requirement: Path) -> List[str]:
     """Parse a requirement files into an array."""
-    requirement_list = []
+    requirement_list = set()
     with requirement.open("r") as data:
         for line in data:
             line = line.strip()
             if not line or line.startswith("#"):
                 continue
-            requirement_list.append(line.split(" ")[-1])
-    return requirement_list
+            requirement_list.add(line.split(" ")[-1])
+    return list(requirement_list)
 
 
 def extract_packages(
@@ -51,3 +71,8 @@ def extract_packages(
     packages_diff = parse_requirements(requirement_diff)
 
     return list(set(packages) - set(packages_diff))
+
+
+def write_requirement(requirement: Path, packages: List[str]) -> None:
+    """Write packages list to a requirement file."""
+    requirement.write_text("\n".join(packages))

requirements.txt

@@ -1,5 +1,5 @@
 click==7.0
 click-pathlib==2019.4.30.2
 requests==2.22.0
-cython==0.29.9
+Cython==0.29.10
 wheel==0.33.4

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup
 
-VERSION = 0.6
+VERSION = 0.7
 
 setup(
     name="builder",