From fd08257bcafd1c75130b4c77941ac59d01dfa28d Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Fri, 24 Jan 2025 01:30:02 -0700 Subject: [PATCH] feat: Update to AB v25.2.30 with python_tool_handle bindings (#71) * Update to AnalysisBase v25.2.30 with columnar CP tools and nanobind bindings to PythonToolHandle. - gitlab-registry.cern.ch/feickert/columnar-athena-containers:analysisbase-25.2.30-latest * Remove https://gitlab.cern.ch/gstark/pycolumnarprototype.git as no longer necessary or useful. * Rebuild lock file. --- Makefile | 8 +++---- docker/Dockerfile | 35 ++++-------------------------- docker/Dockerfile.lockfile-builder | 4 ++-- docker/requirements.lock | 35 ++---------------------------- 4 files changed, 12 insertions(+), 70 deletions(-) diff --git a/Makefile b/Makefile index a41b83d..f9f3d78 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ lock: - docker pull gitlab-registry.cern.ch/atlas/athena/analysisbase:25.2.2 + docker pull gitlab-registry.cern.ch/atlas/athena/analysisbase:25.2.30 docker build \ --file docker/Dockerfile.lockfile-builder \ - --build-arg BASE_IMAGE=gitlab-registry.cern.ch/atlas/athena/analysisbase:25.2.2 \ + --build-arg BASE_IMAGE=gitlab-registry.cern.ch/atlas/athena/analysisbase:25.2.30 \ --tag tmp/analysisbase:rel25-lockfile-builder \ docker docker run \ @@ -19,10 +19,10 @@ lock: rm -r venv' build: - docker pull gitlab-registry.cern.ch/atlas/athena/analysisbase:25.2.2 + docker pull gitlab-registry.cern.ch/atlas/athena/analysisbase:25.2.30 docker build \ --file docker/Dockerfile \ - --build-arg BASE_IMAGE=gitlab-registry.cern.ch/atlas/athena/analysisbase:25.2.2 \ + --build-arg BASE_IMAGE=gitlab-registry.cern.ch/atlas/athena/analysisbase:25.2.30 \ --tag sslhep/analysis-dask-base:debug \ . diff --git a/docker/Dockerfile b/docker/Dockerfile index 514d594..8f2fb7f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,5 @@ -ARG BASE_IMAGE=gitlab-registry.cern.ch/atlas/athena/analysisbase:25.2.2 -FROM ${BASE_IMAGE} as base +ARG BASE_IMAGE=gitlab-registry.cern.ch/feickert/columnar-athena-containers:analysisbase-25.2.30-latest +FROM ${BASE_IMAGE} AS base SHELL [ "/bin/bash", "-c" ] @@ -48,34 +48,6 @@ RUN chmod +x /tini && \ chmod +x /cmd.sh && \ chmod +x /release_setup.sh -# Need to use an additonal directory beyond /usr/AnalysisBase to install -# given how ATLAS CMake works. -# c.f. https://gitlab.cern.ch/gstark/pycolumnarprototype/-/issues/2 -RUN . /release_setup.sh && \ - cd /tmp && \ - git clone \ - --recurse-submodules \ - --branch py_el_tool_test \ - https://gitlab.cern.ch/gstark/pycolumnarprototype.git && \ - cd pycolumnarprototype && \ - cmake \ - -S src \ - -B build && \ - cmake build -LH && \ - cmake \ - --build build \ - --clean-first \ - --parallel "$(nproc --ignore=1)" && \ - DESTDIR=/usr/tools cmake --install build && \ - cd /tmp && \ - rm -rf pycolumnarprototype && \ - echo -e "\n# Set up the PyColumnarPrototype tool:" >> /release_setup.sh && \ - echo -e ". $(find /usr/tools -type f -iname 'setup.sh')" >> /release_setup.sh && \ - echo -e 'echo "Configured PyColumnarPrototype from: ${PyColumnarPrototypeDemo_DIR}"' >> /release_setup.sh && \ - . /release_setup.sh && \ - cvmfs-venv-rebase && \ - python -c 'import PyColumnarPrototype; print(f"{PyColumnarPrototype.column_maker()=}")' - # Always have this be the last edit to /release_setup.sh RUN echo -e "\n# Ensure that the virtual environment is always at the HEAD of PYTHONPATH\ncvmfs-venv-rebase" >> /release_setup.sh @@ -86,7 +58,8 @@ USER atlas RUN echo -e '\n# Activate AnalysisBase environment on login shell\n. /release_setup.sh\n' >> /home/atlas/.bashrc && \ . /release_setup.sh && \ mkdir -p $(jupyter --config) && \ - curl -LsSf https://astral.sh/uv/install.sh | sh + curl -LsSf https://astral.sh/uv/install.sh | sh && \ + python -c 'import python_tool_handle; tool_handle = python_tool_handle.PythonToolHandle(); print(tool_handle)' # $(jupyter --config) should be /home/atlas/.jupyter/ COPY --chown=atlas docker/jupyter_lab_config.py /home/atlas/.jupyter/ diff --git a/docker/Dockerfile.lockfile-builder b/docker/Dockerfile.lockfile-builder index d8eb07c..dba72cd 100644 --- a/docker/Dockerfile.lockfile-builder +++ b/docker/Dockerfile.lockfile-builder @@ -1,5 +1,5 @@ -ARG BASE_IMAGE=gitlab-registry.cern.ch/atlas/athena/analysisbase:25.2.2 -FROM ${BASE_IMAGE} as base +ARG BASE_IMAGE=gitlab-registry.cern.ch/atlas/athena/analysisbase:25.2.30 +FROM ${BASE_IMAGE} AS base SHELL [ "/bin/bash", "-c" ] diff --git a/docker/requirements.lock b/docker/requirements.lock index 9c94c9e..bf8d7b9 100644 --- a/docker/requirements.lock +++ b/docker/requirements.lock @@ -168,10 +168,6 @@ async-lru==2.0.4 \ --hash=sha256:b8a59a5df60805ff63220b2a0c5b5393da5521b113cd5465a44eb037d81a5627 \ --hash=sha256:ff02944ce3c288c5be660c42dbcca0742b32c3b279d6dceda655190240b99224 # via jupyterlab -async-timeout==4.0.3 \ - --hash=sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f \ - --hash=sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028 - # via aiohttp asyncache==0.3.1 \ --hash=sha256:9a1e60a75668e794657489bdea6540ee7e3259c483517b934670db7600bf5035 \ --hash=sha256:ef20a1024d265090dd1e0785c961cf98b9c32cc7d9478973dcf25ac1b80011f5 @@ -854,10 +850,7 @@ distributed==2024.4.2 \ exceptiongroup==1.2.2 \ --hash=sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b \ --hash=sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc - # via - # anyio - # ipython - # kr8s + # via kr8s executing==2.1.0 \ --hash=sha256:8d63781349375b5ebccc3142f4b30350c0cd9c79f921cde38be2be4637e98eaf \ --hash=sha256:8ea27ddd260da8150fa5a708269c4a10e76161e2496ec3e587da9e3c0fe4b9ab @@ -1092,17 +1085,6 @@ importlib-metadata==8.4.0 \ # via # awkward # dask - # jupyter-client - # jupyter-lsp - # jupyter-server-proxy - # jupyterlab - # jupyterlab-server - # nbconvert - # servicex -importlib-resources==6.4.4 \ - --hash=sha256:20600c8b7361938dc0bb2d5ec0297802e575df486f5a544fa414da65e13721f7 \ - --hash=sha256:dda242603d1c9cd836c3368b1174ed74cb4049ecd209e7a1a0104620c18c5c11 - # via matplotlib ipykernel==6.29.5 \ --hash=sha256:afdb66ba5aa354b09b91379bac28ae4afebbb30e8b39510c9690afb7a10421b5 \ --hash=sha256:f093a22c4a40f8828f8e330a9c297cb93dcab13bd9678ded6de8e5cf81c56215 @@ -2798,10 +2780,6 @@ toml==0.10.2 \ # via # coffea # jupytext -tomli==2.0.1 \ - --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ - --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f - # via jupyterlab toolz==0.12.1 \ --hash=sha256:d22731364c07d72eea0a0ad45bafb2c2937ab6fd38a3507bf55eae8744aa7d85 \ --hash=sha256:ecca342664893f177a13dac0e6b41cbd8ac25a358e5f215316d43e2100224f4d @@ -2871,18 +2849,11 @@ typing-extensions==4.12.2 \ --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 # via - # anyio - # async-lru - # awkward # dask-awkward - # hist - # ipython # kopf # pydantic # pydantic-core - # servicex # typer - # uproot tzdata==2024.1 \ --hash=sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd \ --hash=sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252 @@ -3185,9 +3156,7 @@ zict==3.0.0 \ zipp==3.20.1 \ --hash=sha256:9960cd8967c8f85a56f920d5d507274e74f9ff813a0ab8889a5b5be2daf44064 \ --hash=sha256:c22b14cc4763c5a5b04134207736c107db42e9d3ef2d9779d465f5f1bcba572b - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata zstandard==0.23.0 \ --hash=sha256:034b88913ecc1b097f528e42b539453fa82c3557e414b3de9d5632c80439a473 \ --hash=sha256:0a7f0804bb3799414af278e9ad51be25edf67f78f916e08afdb983e74161b916 \