From 10d7c3fdc61636cfd2e04a3e32701ecb13ed235a Mon Sep 17 00:00:00 2001 From: Olof Kindgren Date: Fri, 31 Jan 2025 16:08:54 +0100 Subject: [PATCH] Add support for custom build runners --- doc/user/build_runners.rst | 45 ++++++++++++++++++++++++++++++++ doc/user/index.rst | 2 ++ edalize/build_runners/make.py | 48 +++++++++++++++++++++++++++++++++++ edalize/flows/edaflow.py | 20 ++++++++++++--- pyproject.toml | 2 +- 5 files changed, 112 insertions(+), 5 deletions(-) create mode 100644 doc/user/build_runners.rst create mode 100644 edalize/build_runners/make.py diff --git a/doc/user/build_runners.rst b/doc/user/build_runners.rst new file mode 100644 index 000000000..28636e94f --- /dev/null +++ b/doc/user/build_runners.rst @@ -0,0 +1,45 @@ +Build runners +============= + +Edalize will by default create a Makefile which it subsequently executes to offload the process of keeping track of what needs to be rebuilt when source files change. In Edalize we call make a build runner since it runs the build process. + +In some cases it can be necessary to augment or completely replace the makefile generation, e.g. to initialize some environment before launching or to integrate with a custom build system. + +It is therefore possible to replace the default build runner with a custom tool by creating a new module under the `edalize.build_runner` namespace. In this module Edalize expects to find a class with the same name as the module but capitalized. This class should have the following three functions defined. + +**__init__(self, flow_options)** Constructor that also receives the flow_options defined. + +**get_build_command(self)** Returns a tuple where the first element is the command to launch (e.g. `make` if we are executing a Makefile) and the second element is a list of options to send to the command (e.g. `["-j4", "-d"]` for a `make` process). + +**write(self, commands: EdaCommands, work_root: Path)** Write any required files needed for building. For the `make` build runner, this creates the actual Makefile. + +Below is an example of a build runner that extends the `make` build runner to copy the build tree to a server over ssh and the execute it from there. + + +Build runner examplee:: + + from typing import List + from pathlib import Path + + from edalize.build_runners.make import Make + from edalize.utils import EdaCommands + + class Sshmake(Make): + + def __init__(self, flow_options): + super().__init__(flow_options) + self.build_host = "5446.54.40.138" + + def get_build_command(self): + return ("sh", ["launchme.sh"]) + + def write(self, commands: EdaCommands, work_root: Path): + # Write Makefile + super().write(commands, work_root) + + # Write launcher script that copies files to build host and runs it + outfile = work_root / Path("launchme.sh") + with open(outfile, "w") as f: + f.write("#Auto generated by Edalize\n\n") + f.write(f"scp -r {work_root} {self.build_host}\n") + f.write(f"ssh {self.build_host} make " + ' '.join(self.build_options)+ "\n") diff --git a/doc/user/index.rst b/doc/user/index.rst index e6ecdec06..c03dabd14 100644 --- a/doc/user/index.rst +++ b/doc/user/index.rst @@ -11,3 +11,5 @@ The Edalize flow is divided into three stages called `configure`, `build` and `r .. include:: configure.rst .. include:: build.rst .. include:: run.rst + +.. include:: build_runners.rst diff --git a/edalize/build_runners/make.py b/edalize/build_runners/make.py new file mode 100644 index 000000000..f1eba5b3b --- /dev/null +++ b/edalize/build_runners/make.py @@ -0,0 +1,48 @@ +from typing import List +from pathlib import Path + +from edalize.utils import EdaCommands + + +class Make(object): + def __init__(self, flow_options): + self.build_options = flow_options.get("flow_make_options", []) + + def get_build_command(self): + return ("make", self.build_options) + + def write(self, commands: EdaCommands, work_root: Path): + outfile = work_root / Path("Makefile") + with open(outfile, "w") as f: + f.write("#Auto generated by Edalize\n\n") + for v in commands.variables: + f.write(v + "\n") + if commands.variables: + f.write("\n\n") + if not commands.default_target: + raise RuntimeError("Internal Edalize error. Missing default target") + + f.write(f"all: {commands.default_target}\n") + + for c in commands.commands: + f.write(f"\n{' '.join(c.targets)}:") + for d in c.depends: + f.write(" " + d) + if c.order_only_deps: + f.write(" |") + for d in c.order_only_deps: + f.write(" " + d) + + f.write("\n") + + env_prefix = "" + if c.variables: + env_prefix += "env " + for key, value in c.variables.items(): + env_prefix += f"{key}={value} " + + for command in c.commands: + if command: + f.write( + f"\t$(EDALIZE_LAUNCHER) {env_prefix}{' '.join([str(x) for x in command])}\n" + ) diff --git a/edalize/flows/edaflow.py b/edalize/flows/edaflow.py index 9fbdeaec9..d48c2a440 100644 --- a/edalize/flows/edaflow.py +++ b/edalize/flows/edaflow.py @@ -129,6 +129,10 @@ def get_nodes(self): class Edaflow(object): FLOW_OPTIONS = { + "build_runner": { + "type": "str", + "desc": "Tool to execute the build graph (Defaults to make)", + }, "frontends": { "type": "str", "desc": "Tools to run before main flow", @@ -307,6 +311,15 @@ def __init__(self, edam, work_root, verbose=False): self.set_run_command() self.add_scripts("run", "post_run") + # Initialize build runner + _br = self.flow_options.get("build_runner", "make") + try: + self.build_runner = getattr( + import_module(f"edalize.build_runners.{_br}"), _br.capitalize() + )(self.flow_options) + except ModuleNotFoundError: + raise RuntimeError(f"Could not find build runner '{_br}'") + def set_run_command(self): self.commands.add([], ["run"], ["pre_run"]) @@ -317,7 +330,7 @@ def configure(self): node.inst.configure() # Write out execution file - self.commands.write(os.path.join(self.work_root, "Makefile")) + self.build_runner.write(self.commands, self.work_root) def _run_tool(self, cmd, args=[], cwd=None, quiet=False, env={}): logger.debug("Running " + cmd) @@ -353,9 +366,8 @@ def _run_tool(self, cmd, args=[], cwd=None, quiet=False, env={}): return cp.returncode, cp.stdout, cp.stderr def build(self): - # FIXME: Get run command (e.g. make, ninja, cloud thingie..) from self.commands - make_options = self.flow_options.get("flow_make_options", []) - self._run_tool("make", args=make_options, cwd=self.work_root) + (cmd, args) = self.build_runner.get_build_command() + self._run_tool(cmd, args=args, cwd=self.work_root) # Most flows won't have a run phase def run(self, args=None): diff --git a/pyproject.toml b/pyproject.toml index 1dd1d37f4..8591e278a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,4 +39,4 @@ script-files = ["scripts/el_docker"] write_to = "edalize/version.py" [tool.setuptools.packages.find] -include = ["edalize", "edalize.tools", "edalize.flows"] +include = ["edalize", "edalize.tools", "edalize.flows", "edalize.build_runners"]