From 481a3f22191ddf6038684967afda66f5653f8544 Mon Sep 17 00:00:00 2001 From: Matthew Woehlke Date: Thu, 9 Jan 2025 14:24:57 -0500 Subject: [PATCH] Improve how we manage which PDM to use Move our PDM version pin to a separate file. This both makes it easier to manage updates to the same and also allows the pinned version to be usable outside of the venv_sync script. Teach venv_upgrade to use PDM itself to generate this file, which accomplishes the outstanding goal of being able to pin both PDM and its dependencies. Also, modify venv_sync to tell PDM to not complain about newer versions. --- setup/BUILD.bazel | 1 + setup/python/pyproject.toml | 3 ++ setup/python/requirements.txt | 12 ++++++++ setup/python/requirements.txt.in | 13 +++++++++ tools/workspace/python/repository.bzl | 10 +++++-- tools/workspace/python/venv_sync | 14 +++++---- tools/workspace/python/venv_upgrade | 42 ++++++++++++++++++++++++--- 7 files changed, 82 insertions(+), 13 deletions(-) create mode 100644 setup/python/requirements.txt create mode 100644 setup/python/requirements.txt.in diff --git a/setup/BUILD.bazel b/setup/BUILD.bazel index 16b9910c0391..8e5fe09d8571 100644 --- a/setup/BUILD.bazel +++ b/setup/BUILD.bazel @@ -13,6 +13,7 @@ exports_files([ "mac/source_distribution/Brewfile-maintainer-only", "python/pdm.lock", "python/pyproject.toml", + "python/requirements.txt", "ubuntu/binary_distribution/packages-jammy.txt", "ubuntu/source_distribution/packages-jammy.txt", "ubuntu/source_distribution/packages-jammy-clang.txt", diff --git a/setup/python/pyproject.toml b/setup/python/pyproject.toml index d042440a4d16..c47779c775b7 100644 --- a/setup/python/pyproject.toml +++ b/setup/python/pyproject.toml @@ -20,6 +20,9 @@ dependencies = [ ] [dependency-groups] +pdm = [ + "pdm", +] test = [ "flask", "six", diff --git a/setup/python/requirements.txt b/setup/python/requirements.txt new file mode 100644 index 000000000000..63a8fe0923d6 --- /dev/null +++ b/setup/python/requirements.txt @@ -0,0 +1,12 @@ +# This file does NOT describe the requirements for Drake itself, but for +# setting up the Python virtual environment that Drake will manage. It is used +# by Drake's @python, and also when preparing a virtual environment to support +# a Drake installation. +# +# This file will be generated by tools/workspace/python/venv_upgrade. +# +# The present contents are adequate to bootstrap the PDM installation, after +# which (the first subsequent time venv_upgrade is run) PDM itself will +# generate a complete list of its dependencies with pinned versions. + +pdm==2.22.0 diff --git a/setup/python/requirements.txt.in b/setup/python/requirements.txt.in new file mode 100644 index 000000000000..da5f3eefe443 --- /dev/null +++ b/setup/python/requirements.txt.in @@ -0,0 +1,13 @@ +# This file does NOT describe the requirements for Drake itself, but for +# setting up the Python virtual environment that Drake will manage. It is used +# by Drake's @python, and also when preparing a virtual environment to support +# a Drake installation. +# +# This file will be generated by tools/workspace/python/venv_upgrade. +# +# This version serves two purposes. First, it can be used to bootstrap the PDM +# installation (the initial versions of PDM and dependencies will not be +# pinned). Second, the above text is used as the template for the generated +# version. + +pdm diff --git a/tools/workspace/python/repository.bzl b/tools/workspace/python/repository.bzl index 09da29331301..e1fed35f9d3d 100644 --- a/tools/workspace/python/repository.bzl +++ b/tools/workspace/python/repository.bzl @@ -114,9 +114,13 @@ def _prepare_venv(repo_ctx, python): if os_name != "mac os x": return python - # Locate the lock file and mark it to be monitored for changes. - pylock = repo_ctx.path(Label("@drake//setup:python/pdm.lock")).realpath - repo_ctx.watch(pylock) + # Locate lock files and mark them to be monitored for changes. + requirements = repo_ctx.path( + Label("@drake//setup:python/requirements.txt"), + ).realpath + pdmlock = repo_ctx.path(Label("@drake//setup:python/pdm.lock")).realpath + repo_ctx.watch(requirements) + repo_ctx.watch(pdmlock) # Choose which dependencies to install. if repo_ctx.attr.requirements_flavor == "test": diff --git a/tools/workspace/python/venv_sync b/tools/workspace/python/venv_sync index b04f964fb0b2..89da3b0e8822 100755 --- a/tools/workspace/python/venv_sync +++ b/tools/workspace/python/venv_sync @@ -53,18 +53,17 @@ fi # Place the venv(s) in a sibling directory to the output base. That should be a # suitable disk location for build artifacts, but without polluting the actual # output base that Bazel owns. +readonly drake_root="$(cd "$(dirname $0)/../../.." && pwd)" readonly bazel_output_base="$(cd "${repository}/../.." && pwd)" readonly drake_python="${bazel_output_base}.drake_python" mkdir -p "${drake_python}" # Install PDM into a virtual environment. We segregate PDM from the environment # it is managing, so that changes to the managed environment cannot break PDM. +readonly setup="${drake_root}/setup/python" readonly venv_pdm="${drake_python}/venv.pdm" mkvenv "${venv_pdm}" -# TODO(jeremy.nimmer) Ideally, we also would pin all of the dependencies of -# PDM here, but it's not obvious to me how to do that in a way which is easy to -# upgrade/re-pin over time. -"${venv_pdm}/bin/pip" install -U pdm==2.22.0 +"${venv_pdm}/bin/pip" install -U -r "${setup}/requirements.txt" # Don't nag about new versions of PDM; we'll update the above pin when we want # to use something newer, and otherwise want to use the pinned version, so @@ -72,13 +71,16 @@ mkvenv "${venv_pdm}" export PDM_CHECK_UPDATE=0 # Prepare the PDM "project directory". -readonly drake_root="$(cd "$(dirname $0)/../../.." && pwd)" -readonly setup="${drake_root}/setup/python" readonly project="${drake_python}/project" mkdir -p "${project}" ln -nsf "${setup}/pyproject.toml" "${project}/pyproject.toml" ln -nsf "${setup}/pdm.lock" "${project}/pdm.lock" +# Don't nag about new versions of PDM; venv_upgrade will check for those. +# Aside from that, we want to use the pinned version, so being informed about +# newer versions is just noise. +export PDM_CHECK_UPDATE=0 + # Prepare the venv that will hold Drake's requirements. readonly venv_drake="${drake_python}/venv.drake" mkvenv "${venv_drake}" diff --git a/tools/workspace/python/venv_upgrade b/tools/workspace/python/venv_upgrade index cd13ec8c4836..313f12d1fc3d 100755 --- a/tools/workspace/python/venv_upgrade +++ b/tools/workspace/python/venv_upgrade @@ -3,13 +3,23 @@ # Drake script to upgrade our requirements lockfile. # - Users should make edits to pyproject.toml, and then # - run this script to compile pyproject.toml to pdm.lock. +# +# The --reuse option may be given to apply changes to pyproject.toml while +# attempting to preserve existing version pins. +# +# This script should also be run periodically to update pinned Python packages +# to the latest available versions. set -eux -o pipefail commit= files_to_commit=() +lock_args=() while [ "${1:-}" != "" ]; do case "$1" in + --reuse) + lock_args+=( --update-reuse ) + ;; --commit) commit=1 ;; @@ -34,11 +44,19 @@ check_working_tree() { fi } +update_requirements() { + sed $'/will be generated/{s/will be/is/\nq\n}' < "$1.in" > "$1" + "${venv_pdm}/bin/pdm" export -p "${project}" -G pdm --no-default | + grep -vE '^#' >> "$1" +} + # Chdir to the Drake root. cd "$(dirname $0)/../../.." readonly drake_python="$(bazel info output_base).drake_python" readonly project="${drake_python}/project" readonly venv_pdm="${drake_python}/venv.pdm" +readonly python="${venv_pdm}/bin/python" +readonly workspace="$(pwd)/tools/workspace/python" # If committing, check that the working tree is clean (enough). Note that the # lock file is ignored. @@ -47,11 +65,27 @@ readonly venv_pdm="${drake_python}/venv.pdm" # Ensure venv exists. bazel fetch @python --force -# Remove and recreate the lock file. +# Update the lock file. readonly lockfile="./setup/python/pdm.lock" -rm -f "${lockfile}" -"${venv_pdm}/bin/pdm" lock -p "${project}" -L "${lockfile}" -files_to_commit+=($(git diff --name-only HEAD -- "${lockfile}")) +lock_args+=( -d -p "${project}" -L "${lockfile}" ) +"${venv_pdm}/bin/pdm" lock "${lock_args[@]}" + +# Update pins for PDM itself +readonly requirements="./setup/python/requirements.txt" +update_requirements "${requirements}" +files_to_commit+=( "$(git diff --name-only HEAD -- "${requirements}")" ) + +if [ ${#files_to_commit[@]} -gt 0 ]; then + # PDM needs to be updated. + bazel fetch @python + + # Update the lock files (again). + update_requirements "${requirements}" + "${venv_pdm}/bin/pdm" lock "${lock_args[@]}" +fi + +# Check for changes to the lock file. +files_to_commit+=( "$(git diff --name-only HEAD -- "${lockfile}")" ) # If committing, do the commit. if [ -n "${commit}" ]; then