[bazel,python] vendor in requirements.bzl

Our current version of rules_python autogenerates the `requirements.bzl` file from a `requirements.txt` lock file. However, this is awkward if this project is ever a dependency of another downstream bazel project, as shown here: bazelbuild/rules_python#608 To circumvent this issue, rules_python provides a mechanism to vendor in the requirements.bzl file into the repo, which this commit does. Signed-off-by: Tim Trippel <[email protected]>
lowRISC · Jan 31, 2025 · b2c2529 · b2c2529
1 parent 7ea4040
commit b2c2529
Show file tree

Hide file tree

Showing 30 changed files with 561 additions and 30 deletions.
diff --git a/WORKSPACE b/WORKSPACE
@@ -20,7 +20,7 @@ load("//third_party/python:deps.bzl", "python_deps")
 python_deps()
 load("//third_party/python:pip.bzl", "pip_deps")
 pip_deps()
-load("@ot_python_deps//:requirements.bzl", install_ot_python_deps="install_deps")
+load("//third_party/python:requirements.bzl", install_ot_python_deps="install_deps")
 install_ot_python_deps()
 
 # Google/Bazel dependencies.  This needs to be after Python initialization

diff --git a/doc/contributing/sw/adding_python_depedencies.md b/doc/contributing/sw/adding_python_depedencies.md
@@ -1,8 +1,21 @@
-# Adding Python Dependencies to the Project
+# Python Dependency Management
 
 To comply with software supply chain security requirements of various project partner organizations, we pin our Python packages to specific versions, and provide hashes for each dependency (including transitive dependencies).
 We accomplish this with the help of the `pip-compile` tool, which is part of the `pip-tools` package.
 
+Additionally, to make it easier for downstream Bazel projects to dependend on this Bazel project (`@lowrisc_opentitan`), we vendor in the autogenerated `requirements.bzl` file produced by `rules_python`, see the `rules_python` [documentation](https://rules-python.readthedocs.io/en/0.39.0/pypi-dependencies.html#vendoring-the-requirements-bzl-file) for more details.
+
+# Adding Python Dependencies to the Project
 If you need to add another Python package to the project, do so by:
-1. adding the package and version number to the `python-requirements.in` file, in the form of `<package>==<version>`, and
-1. run the script `util/sh/scripts/gen-python-requirements.sh`, which will auto-generate the updated `python-requirements.txt` file.
+1. adding the package and version number to the `python-requirements.in` file, in the form of `<package>==<version>`,
+1. running the script `util/sh/scripts/gen-python-requirements.sh`, which will auto-generate the updated `python-requirements.txt` file as shown below, and
+1. running the `//third_party/python:vendor_requirements` Bazel target which will update the vendored in `requirements.bzl` file as shown below.
+
+```sh
+# Update autogenerated python package management files.
+./util/sh/scripts/gen-python-requirements.sh > python-requirements.txt
+bazel run //third_party/python:vendor_requirements
+
+# Check that the package management files are up to date.
+bazel test //third_party/python/...
+```
diff --git a/hw/ip/otbn/dv/otbnsim/sim/BUILD b/hw/ip/otbn/dv/otbnsim/sim/BUILD
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 load("@rules_python//python:defs.bzl", "py_library")
-load("@ot_python_deps//:requirements.bzl", "requirement")
+load("//third_party/python:requirements.bzl", "requirement")
 
 package(default_visibility = ["//visibility:public"])
 

diff --git a/hw/ip/otbn/util/BUILD b/hw/ip/otbn/util/BUILD
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 load("@rules_python//python:defs.bzl", "py_binary")
-load("@ot_python_deps//:requirements.bzl", "requirement")
+load("//third_party/python:requirements.bzl", "requirement")
 
 package(default_visibility = ["//visibility:public"])
 

diff --git a/hw/ip/rom_ctrl/util/BUILD b/hw/ip/rom_ctrl/util/BUILD
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
-load("@ot_python_deps//:requirements.bzl", "requirement")
+load("//third_party/python:requirements.bzl", "requirement")
 
 package(default_visibility = ["//visibility:public"])
 

diff --git a/rules/scripts/BUILD b/rules/scripts/BUILD
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 load("@rules_python//python:defs.bzl", "py_test")
-load("@ot_python_deps//:requirements.bzl", "requirement")
+load("//third_party/python:requirements.bzl", "requirement")
 
 package(default_visibility = ["//visibility:public"])
 

diff --git a/sw/device/silicon_creator/lib/sigverify/sigverify_tests/BUILD b/sw/device/silicon_creator/lib/sigverify/sigverify_tests/BUILD
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 load("//rules:autogen.bzl", "autogen_cryptotest_header")
-load("@ot_python_deps//:requirements.bzl", "requirement")
+load("//third_party/python:requirements.bzl", "requirement")
 load(
     "//rules/opentitan:defs.bzl",
     "fpga_params",

diff --git a/sw/device/silicon_creator/lib/sigverify/sphincsplus/test/BUILD b/sw/device/silicon_creator/lib/sigverify/sphincsplus/test/BUILD
@@ -15,7 +15,7 @@ load(
     "opentitan_test",
     "verilator_params",
 )
-load("@ot_python_deps//:requirements.bzl", "requirement")
+load("//third_party/python:requirements.bzl", "requirement")
 
 package(default_visibility = ["//visibility:public"])
 

diff --git a/sw/device/tests/crypto/BUILD b/sw/device/tests/crypto/BUILD
@@ -16,7 +16,7 @@ load(
     "silicon_params",
     "verilator_params",
 )
-load("@ot_python_deps//:requirements.bzl", "requirement")
+load("//third_party/python:requirements.bzl", "requirement")
 
 package(default_visibility = ["//visibility:public"])
 

diff --git a/sw/device/tests/crypto/testvectors/wycheproof/BUILD b/sw/device/tests/crypto/testvectors/wycheproof/BUILD
@@ -4,7 +4,7 @@
 
 package(default_visibility = ["//visibility:public"])
 
-load("@ot_python_deps//:requirements.bzl", "requirement")
+load("//third_party/python:requirements.bzl", "requirement")
 
 py_binary(
     name = "rsa_3072_verify_parse_testvectors",

diff --git a/sw/host/cryptotest/testvectors/parsers/BUILD b/sw/host/cryptotest/testvectors/parsers/BUILD
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
-load("@ot_python_deps//:requirements.bzl", "requirement")
+load("//third_party/python:requirements.bzl", "requirement")
 
 package(default_visibility = ["//visibility:public"])
 

diff --git a/sw/host/provisioning/orchestrator/src/BUILD b/sw/host/provisioning/orchestrator/src/BUILD
@@ -2,7 +2,7 @@
 # Licensed under the Apache License, Version 2.0, see LICENSE for details.
 # SPDX-License-Identifier: Apache-2.0
 
-load("@ot_python_deps//:requirements.bzl", "requirement")
+load("//third_party/python:requirements.bzl", "requirement")
 load("@rules_python//python:defs.bzl", "py_binary", "py_library")
 
 package(default_visibility = ["//visibility:public"])

diff --git a/sw/host/provisioning/orchestrator/tests/BUILD b/sw/host/provisioning/orchestrator/tests/BUILD
@@ -2,7 +2,7 @@
 # Licensed under the Apache License, Version 2.0, see LICENSE for details.
 # SPDX-License-Identifier: Apache-2.0
 
-load("@ot_python_deps//:requirements.bzl", "requirement")
+load("//third_party/python:requirements.bzl", "requirement")
 load("@rules_python//python:defs.bzl", "py_test")
 load(
     "//sw/device/silicon_creator/manuf/base:provisioning_inputs.bzl",

diff --git a/third_party/python/BUILD b/third_party/python/BUILD
@@ -2,8 +2,66 @@
 # Licensed under the Apache License, Version 2.0, see LICENSE for details.
 # SPDX-License-Identifier: Apache-2.0
 
+load("@bazel_skylib//rules:build_test.bzl", "build_test")
+load("@bazel_skylib//rules:diff_test.bzl", "diff_test")
+load("@bazel_skylib//rules:write_file.bzl", "write_file")
+load("//third_party/python:requirements.bzl", "all_data_requirements", "all_requirements", "all_whl_requirements")
+
 package(default_visibility = ["//visibility:public"])
 
 exports_files([
     "gen_requirements.sh",
 ])
+
+# The requirements.bzl file is using the hub repo to access packages via the
+# `requirement` macro and when the requirements.bzl is vendored, the hub
+# repo won't be present. As a result, we have to adjust the label scheme in
+# the requirements.bzl to make sure that they continue to work.
+genrule(
+    name = "requirements_bzl",
+    srcs = ["@ot_python_deps//:requirements.bzl"],
+    outs = ["requirements.clean.bzl"],
+    cmd = " | ".join([
+        "cat $<",
+        # Substitute the name of the hub to ensure that the dependencies do
+        # not require the hub repo initialized in the WORKSPACE.
+        "sed -e 's/ot_python_deps/ot_python_deps_vendored/g'",
+        # Change the labels from using the hub repo to using the spoke repos
+        # directly.
+        "sed -e 's|//\\([^:]*\\):pkg|_\\1//:pkg|g'",
+        "sed -e 's|//\\([^:]*\\):whl|_\\1//:whl|g'",
+        "sed -e 's|//\\([^:]*\\):data|_\\1//:data|g'",
+        # Change the convenience macros to use the same naming.
+        "sed -e 's|//{}:{}|_{}//:{}|g' >$@",
+    ]),
+)
+
+write_file(
+    name = "gen_update",
+    out = "update.sh",
+    content = [
+        "#!/usr/bin/env bash",
+        "cd $BUILD_WORKSPACE_DIRECTORY",
+        "cp -fv bazel-bin/third_party/python/requirements.clean.bzl third_party/python/requirements.bzl",
+    ],
+)
+
+sh_binary(
+    name = "vendor_requirements",
+    srcs = ["update.sh"],
+    data = [":requirements_bzl"],
+)
+
+# Similarly ensures that the requirements.bzl file is updated based on the
+# requirements.txt lockfile.
+diff_test(
+    name = "test_vendored",
+    failure_message = "Please run:  bazel run //third_party/python:vendor_requirements",
+    file1 = "requirements.bzl",
+    file2 = "requirements_bzl",
+)
+
+build_test(
+    name = "test_requirement_lists",
+    targets = all_requirements + all_whl_requirements + all_data_requirements,
+)