From addf694162fe929b2492e870588b5b3944097a97 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Sun, 2 Jun 2019 12:01:56 +0200 Subject: [PATCH 1/5] Bump version 0.7 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 10b8b314..2aec1d71 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup -VERSION = 0.6 +VERSION = 0.7 setup( name="builder", From 64debbb94aed046ae567bd6f693becd0ec314b62 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Sun, 2 Jun 2019 14:01:25 +0200 Subject: [PATCH 2/5] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fb96ec6b..e5b15999 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -103,11 +103,13 @@ jobs: - script: sudo docker pull homeassistant/$(buildArch)-wheels:$(versionWheels) displayName: 'Install wheels builder' - script: | + curl -s -o requirements_diff.txt https://raw.githubusercontent.com/home-assistant/hassio-wheels/master/requirements.txt sudo docker run --rm -v $(pwd):/data:ro -v $(pwd)/.ssh:/root/.ssh:rw \ homeassistant/$(buildArch)-wheels:$(versionWheels) \ --apk "build-base;libffi-dev;openssl-dev" \ - --index https://wheels.hass.io \ + --index $(wheelsIndex) \ --requirement requirements.txt \ + --requirement-diff requirements_diff.txt \ --upload rsync \ --remote wheels@$(wheelsHost):/opt/wheels displayName: 'Run wheels build' From bc6f774757e8d1c4fd9910e147989c472848b06b Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Sun, 2 Jun 2019 14:01:47 +0200 Subject: [PATCH 3/5] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e5b15999..f5ceced7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -88,7 +88,8 @@ jobs: sudo apt-get update sudo apt-get install -y --no-install-recommends \ qemu-user-static \ - binfmt-support + binfmt-support \ + curl sudo mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc sudo update-binfmts --enable qemu-arm From 8189e8a5a91077edb8a2d37ccf941366463efc6d Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Sun, 2 Jun 2019 21:14:59 +0200 Subject: [PATCH 4/5] Fix requirement (#14) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 851ca1e5..d366c366 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ click==7.0 click-pathlib==2019.4.30.2 requests==2.22.0 -cython==0.29.9 +Cython==0.29.10 wheel==0.33.4 From 78fb7a8d27fe0f98a976facbf308d4b07f80b43c Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Mon, 3 Jun 2019 11:27:24 +0200 Subject: [PATCH 5/5] Allow to build all in once (#15) --- builder/__main__.py | 39 ++++++++++++++++++++++++++++----------- builder/pip.py | 33 +++++++++++++++++++++++++++++---- 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/builder/__main__.py b/builder/__main__.py index f4cfb5f1..afbf2481 100644 --- a/builder/__main__.py +++ b/builder/__main__.py @@ -10,12 +10,17 @@ from builder.apk import install_apks from builder.infra import create_wheels_folder, create_wheels_index -from builder.pip import build_wheels, extract_packages +from builder.pip import ( + build_wheels_package, + build_wheels_requirement, + write_requirement, + extract_packages, +) from builder.upload import run_upload from builder.utils import check_url -@click.command() +@click.command("builder") @click.option("--apk", default="build-base", help="APKs they are needed to build this.") @click.option("--index", required=True, help="Index URL of remote wheels repository.") @click.option( @@ -33,13 +38,15 @@ type=click_pathlib.Path(exists=True), help="Python requirement file to calc the different for selective builds.", ) -def builder(apk, index, requirement, upload, remote, requirement_diff): +@click.option( + "--single", default=False, help="Install every package as single requirement." +) +def builder(apk, index, requirement, upload, remote, requirement_diff, single): """Build wheels precompiled for Home Assistant container.""" install_apks(apk) check_url(index) exit_code = 0 - timer = 0 with TemporaryDirectory() as temp_dir: output = Path(temp_dir) @@ -47,17 +54,27 @@ def builder(apk, index, requirement, upload, remote, requirement_diff): wheels_index = create_wheels_index(index) packages = extract_packages(requirement, requirement_diff) - for package in packages: - print(f"Process package: {package}", flush=True) + if single: + timer = 0 + for package in packages: + print(f"Process package: {package}", flush=True) + try: + build_wheels_package(package, wheels_index, wheels_dir) + except CalledProcessError: + exit_code = 109 + + if timer < monotonic(): + run_upload(upload, output, remote) + timer = monotonic() + 900 + else: + temp_requirement = Path("/tmp/wheels_requirement.txt") + write_requirement(temp_requirement, packages) + try: - build_wheels(package, wheels_index, wheels_dir) + build_wheels_requirement(temp_requirement, wheels_index, wheels_dir) except CalledProcessError: exit_code = 109 - if timer < monotonic(): - run_upload(upload, output, remote) - timer = monotonic() + 900 - run_upload(upload, output, remote) sys.exit(exit_code) diff --git a/builder/pip.py b/builder/pip.py index 748d2600..6d94969a 100644 --- a/builder/pip.py +++ b/builder/pip.py @@ -6,7 +6,7 @@ from typing import List, Optional -def build_wheels(package: str, index: str, output: Path) -> None: +def build_wheels_package(package: str, index: str, output: Path) -> None: """Build wheels from a requirements file into output.""" cpu = os.cpu_count() or 4 @@ -26,16 +26,36 @@ def build_wheels(package: str, index: str, output: Path) -> None: result.check_returncode() +def build_wheels_requirement(requirement: Path, index: str, output: Path) -> None: + """Build wheels from a requirements file into output.""" + cpu = os.cpu_count() or 4 + + # Modify speed + build_env = os.environ.copy() + build_env["MAKEFLAGS"] = f"-j{cpu}" + + result = subprocess.run( + f"pip3 wheel --progress-bar ascii --wheel-dir {output} --find-links {index} --requirement {requirement}", + shell=True, + stdout=sys.stdout, + stderr=sys.stderr, + env=build_env, + ) + + # Check result of program + result.check_returncode() + + def parse_requirements(requirement: Path) -> List[str]: """Parse a requirement files into an array.""" - requirement_list = [] + requirement_list = set() with requirement.open("r") as data: for line in data: line = line.strip() if not line or line.startswith("#"): continue - requirement_list.append(line.split(" ")[-1]) - return requirement_list + requirement_list.add(line.split(" ")[-1]) + return list(requirement_list) def extract_packages( @@ -51,3 +71,8 @@ def extract_packages( packages_diff = parse_requirements(requirement_diff) return list(set(packages) - set(packages_diff)) + + +def write_requirement(requirement: Path, packages: List[str]) -> None: + """Write packages list to a requirement file.""" + requirement.write_text("\n".join(packages))