From bbb07673214258bc405bc1a8bece15be3408507b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Pierret=20=28fepitre=29?= Date: Wed, 26 Feb 2025 17:06:52 +0100 Subject: [PATCH] Add support for using artifacts from another Job Our usecase is for cross-compiling Windows tools --- README.md | 35 ++++++++++++++++-- qubesbuilder/config.py | 57 ++++++++++++++++++++++++++++++ qubesbuilder/executors/__init__.py | 4 +++ qubesbuilder/plugins/__init__.py | 10 ++++++ 4 files changed, 104 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8af429b6..ce494339 100644 --- a/README.md +++ b/README.md @@ -771,8 +771,8 @@ Options available in `builder.yml`: - `packages: bool` --- Component that generate packages (default: True). If set to False (e.g. `builder-rpm`), no `.qubesbuilder` file is allowed. - `verification-mode: str` --- component source code verification mode, supported values are: `signed-tag` (this is default), `less-secure-signed-commits-sufficient`, `insecure-skip-checking`. This option takes precedence over top level `less-secure-signed-commits-sufficient`. - `stages: List[Dict]` --- Allow to override stages options. - - `distribution_name: List[Dict]` -- Allow to override per distribution, stages options. - - `package_set: List[Dict]` -- Allow to override per distribution package set, stages options. + - `distribution_name: List[Dict]` -- Allow to override per distribution, stages options or to provides dependencies. + - `package_set: List[Dict]` -- Allow to override per distribution package set, stages options. - `templates: List[Dict]` -- List of templates you want to build. See example configs for sensible lists. - ``: --- Template name. @@ -859,3 +859,34 @@ For the `fetch` stage, the Qubes executor with disposable template `qubes-builde For the `build` stage of `vm-fc42`, the Podman executor with container image `fedoraimg` will be used. For the `sign` stage, the Qubes executor with disposable template `signing-access-dvm` will be used for both `vm-fc42` and `vm-jammy` For the `prep` stage of `vm-jammy`, the Local executor with base directory `/some/path` will be used. + +### Cross-compile Build + +To perform cross-compilation between distributions, you must declare dependencies using the `needs: List[Dict]` structure within the appropriate distribution stage. +Each dependency is represented as a dictionary with the following keys: + +- `component: str` --- The name of the component. This value is not limited to the top-level component reference; it can reference any available component. +- `distribution: str` --- The name of the target distribution. +- `stage: str` --- The stage name for which the dependency is required. +- `build: str` --- The build reference as provided in a `.qubesbuilder` file. + +For example: + +```yaml +components: + - installer-qubes-os: + host-fc41: + stages: + - build: + needs: + - component: installer-qubes-os + distribution: vm-win10 + stage: build + build: vs2022/installer.sln + - component: installer-qubes-os + distribution: vm-win10 + stage: sign + build: vs2022/installer.sln +``` + +This example shows how to specify multiple dependencies for different stages (build and sign) under a given distribution. diff --git a/qubesbuilder/config.py b/qubesbuilder/config.py index a40d82eb..55e67959 100644 --- a/qubesbuilder/config.py +++ b/qubesbuilder/config.py @@ -37,6 +37,8 @@ DistributionComponentPlugin, ComponentPlugin, TemplatePlugin, + JobReference, + JobDependency, ) from qubesbuilder.template import QubesTemplate @@ -611,6 +613,56 @@ def get_stages(self) -> List[str]: def get_plugin_manager(self): return PluginManager(self.get_plugins_dirs()) + def get_needs( + self, + component: QubesComponent, + dist: QubesDistribution, + stage_name: str, + ): + needs = [] + stages = component.kwargs.get(dist.distribution, {}).get("stages", []) + for stage in stages: + if ( + isinstance(stage, dict) + and next(iter(stage)) == stage_name + and isinstance(stage[stage_name], dict) + ): + for need in stage[stage_name].get("needs", []): + if all( + [ + need.get("component", None), + need.get("distribution", None), + need.get("stage", None), + need.get("build", None), + ] + ): + filtered_components = self.get_components( + [need["component"]] + ) + if not filtered_components: + raise ConfigError( + f"Cannot find dependency component name '{need["component"]}'." + ) + filtered_distributions = self.get_distributions( + [need["distribution"]] + ) + if not filtered_distributions: + raise ConfigError( + f"Cannot find dependency distribution name '{need["distribution"]}'." + ) + needs.append( + JobDependency( + JobReference( + component=filtered_components[0], + dist=filtered_distributions[0], + stage=need["stage"], + template=None, + build=need["build"], + ) + ) + ) + return needs + def get_jobs( self, components: List[QubesComponent], @@ -638,6 +690,11 @@ def get_jobs( ) if not job: continue + job.dependencies += self.get_needs( + component=component, + dist=distribution, + stage_name=stage, + ) jobs.append(job) # ComponentPlugin diff --git a/qubesbuilder/executors/__init__.py b/qubesbuilder/executors/__init__.py index a106cef7..33a0dbe7 100644 --- a/qubesbuilder/executors/__init__.py +++ b/qubesbuilder/executors/__init__.py @@ -78,6 +78,9 @@ def get_repository_dir(self): def get_cache_dir(self): return self.get_builder_dir() / "cache" + def get_dependencies_dir(self): + return self.get_builder_dir() / "dependencies" + @abstractmethod def copy_in(self, *args, **kwargs): pass @@ -102,6 +105,7 @@ def get_placeholders(self): "@BUILD_DIR@": self.get_build_dir(), "@PLUGINS_DIR@": self.get_plugins_dir(), "@DISTFILES_DIR@": self.get_distfiles_dir(), + "@DEPENDENCIES_DIR@": self.get_dependencies_dir(), } def replace_placeholders(self, s: str): diff --git a/qubesbuilder/plugins/__init__.py b/qubesbuilder/plugins/__init__.py index 8697d319..02aeb0c8 100644 --- a/qubesbuilder/plugins/__init__.py +++ b/qubesbuilder/plugins/__init__.py @@ -324,6 +324,16 @@ def default_copy_in(self, plugins_dir: Path, sources_dir: Path): sources_dir, ) ) + if dependency.builder_object == "job": + artifact_path = get_artifact_path( + self.config, dependency.reference + ) + for artifact in artifact_path.parent.iterdir(): + if artifact == artifact_path: + continue + copy_in.append( + (artifact, self.executor.get_dependencies_dir()) + ) return copy_in