From fc6cd0e3e56b6b60af369c7c5dc482a32355c247 Mon Sep 17 00:00:00 2001 From: Jeff Putsch Date: Mon, 18 Dec 2023 14:32:52 -0800 Subject: [PATCH 01/15] install gpg2.22 on centos 7 when installing python --- src/python/README.md | 1 + src/python/devcontainer-feature.json | 23 +- src/python/install.sh | 336 ++++++++++++++++++++++++--- test/python/alma-8.sh | 40 ++++ test/python/alma-9.sh | 40 ++++ test/python/centos-7.sh | 40 ++++ test/python/scenarios.json | 26 ++- 7 files changed, 469 insertions(+), 37 deletions(-) create mode 100755 test/python/alma-8.sh create mode 100755 test/python/alma-9.sh create mode 100755 test/python/centos-7.sh diff --git a/src/python/README.md b/src/python/README.md index 2dd092ae7..41d29336b 100644 --- a/src/python/README.md +++ b/src/python/README.md @@ -18,6 +18,7 @@ Installs the provided version of Python, as well as PIPX, and other common Pytho | version | Select a Python version to install. | string | os-provided | | installTools | Install common Python tools like pylint | boolean | true | | optimize | Optimize Python for performance when compiled (slow) | boolean | false | +| enableShared | Enable building a shared Python library | boolean | true | | installPath | The path where python will be installed. | string | /usr/local/python | | installJupyterlab | Install JupyterLab, a web-based interactive development environment for notebooks | boolean | false | | configureJupyterlabAllowOrigin | Configure JupyterLab to accept HTTP requests from the specified origin | string | - | diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index 7ebdad38a..9b5e5bad9 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -25,13 +25,28 @@ "installTools": { "type": "boolean", "default": true, - "description": "Install common Python tools like pylint" + "description": "Install common Python tools like pylint, override list using 'toolsToInstall'" + }, + "toolsToInstall": { + "type": "string", + "default": "flake8,autopep8,black,yapf,mypy,pydocstyle,pycodestyle,bandit,pipenv,virtualenv,pytest,pylint", + "description": "Comma-separated list of tools to install when installTools is true" + }, + "additionalModules": { + "type": "string", + "default": "", + "description": "Comma-separated list of modules to install via PIP (not pipx)" }, "optimize": { "type": "boolean", "default": false, "description": "Optimize Python for performance when compiled (slow)" }, + "enableShared": { + "type": "boolean", + "default": true, + "description": "Enable building a shared Python library" + }, "installPath": { "type": "string", "default": "/usr/local/python", @@ -71,7 +86,7 @@ } }, "installsAfter": [ - "ghcr.io/devcontainers/features/common-utils", - "ghcr.io/devcontainers/features/oryx" + "ghcr.io/devcontainers/features/common-utils", + "ghcr.io/devcontainers/features/oryx" ] -} +} \ No newline at end of file diff --git a/src/python/install.sh b/src/python/install.sh index e6eefd055..9e9a7d509 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -10,6 +10,7 @@ PYTHON_VERSION="${VERSION:-"latest"}" # 'system' or 'os-provided' checks the base image first, else installs 'latest' INSTALL_PYTHON_TOOLS="${INSTALLTOOLS:-"true"}" OPTIMIZE_BUILD_FROM_SOURCE="${OPTIMIZE:-"false"}" +ENABLE_SHARED_FROM_SOURCE="${ENABLESHARED:-"true"}" PYTHON_INSTALL_PATH="${INSTALLPATH:-"/usr/local/python"}" OVERRIDE_DEFAULT_VERSION="${OVERRIDEDEFAULTVERSION:-"true"}" @@ -26,7 +27,13 @@ CONFIGURE_JUPYTERLAB_ALLOW_ORIGIN="${CONFIGUREJUPYTERLABALLOWORIGIN:-""}" # alongside PYTHON_VERSION, but not set as default. ADDITIONAL_VERSIONS="${ADDITIONALVERSIONS:-""}" -DEFAULT_UTILS=("pylint" "flake8" "autopep8" "black" "yapf" "mypy" "pydocstyle" "pycodestyle" "bandit" "pipenv" "virtualenv" "pytest") +# Comma-separated list of additional tools to be installed via pipx. +IFS="," read -r -a DEFAULT_UTILS <<< "${TOOLSTOINSTALL:-flake8,autopep8,black,yapf,mypy,pydocstyle,pycodestyle,bandit,pipenv,virtualenv,pytest}" + +# Comma-separated list of additional pyhton modules to install via pip. +IFS="," read -r -a PIP_MODULES <<< "${ADDITIONALMODULES:-}" + + PYTHON_SOURCE_GPG_KEYS="64E628F8D684696D B26995E310250568 2D347EA6AA65421D FB9921286F5E1540 3A5CA953F73C700D 04C367C218ADD4FF 0EDDC5F26A45C816 6AF053F07D9DC8D2 C9BE28DEE6DF025C 126EB563A74B06BF D9866941EA5BBD71 ED9D77D5 A821E680E5FA6305" GPG_KEY_SERVERS="keyserver hkp://keyserver.ubuntu.com keyserver hkp://keyserver.ubuntu.com:80 @@ -37,14 +44,58 @@ KEYSERVER_PROXY="${HTTPPROXY:-"${HTTP_PROXY:-""}"}" set -e -# Clean up -rm -rf /var/lib/apt/lists/* - if [ "$(id -u)" -ne 0 ]; then echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' exit 1 fi +# Bring in ID, ID_LIKE, VERSION_ID, VERSION_CODENAME +. /etc/os-release +# Get an adjusted ID independent of distro variants +MAJOR_VERSION_ID=$(echo ${VERSION_ID} | cut -d . -f 1) +if [ "${ID}" = "debian" ] || [ "${ID_LIKE}" = "debian" ]; then + ADJUSTED_ID="debian" +elif [[ "${ID}" = "rhel" || "${ID}" = "fedora" || "${ID}" = "mariner" || "${ID_LIKE}" = *"rhel"* || "${ID_LIKE}" = *"fedora"* || "${ID_LIKE}" = *"mariner"* ]]; then + ADJUSTED_ID="rhel" + if [[ "${ID}" = "rhel" ]] || [[ "${ID}" = *"alma"* ]] || [[ "${ID}" = *"rocky"* ]]; then + VERSION_CODENAME="rhel${MAJOR_VERSION_ID}" + else + VERSION_CODENAME="${ID}${MAJOR_VERSION_ID}" + fi +else + echo "Linux distro ${ID} not supported." + exit 1 +fi + +# Setup INSTALL_CMD & PKG_MGR_CMD +if type apt-get > /dev/null 2>&1; then + PKG_MGR_CMD=apt-get + INSTALL_CMD="${PKG_MGR_CMD} -y install --no-install-recommends" +elif type dnf > /dev/null 2>&1; then + PKG_MGR_CMD=dnf + INSTALL_CMD="${PKG_MGR_CMD} -y install --refresh --best --nodocs --noplugins --setopt=install_weak_deps=0" +elif type microdnf > /dev/null 2>&1; then + PKG_MGR_CMD=microdnf + INSTALL_CMD="${PKG_MGR_CMD} -y install --refresh --best --nodocs --noplugins --setopt=install_weak_deps=0" +else + PKG_MGR_CMD=yum + INSTALL_CMD="${PKG_MGR_CMD} -y install --noplugins --setopt=install_weak_deps=0" +fi + +# Clean up +clean_up() { + case ${ADJUSTED_ID} in + debian) + rm -rf /var/lib/apt/lists/* + ;; + rhel) + rm -rf /var/cache/dnf/* /var/cache/yum/* + ;; + esac +} +clean_up + + # Ensure that login shells get the correct path if the user updated the PATH using ENV. rm -f /etc/profile.d/00-restore-env.sh echo "export PATH=${PATH//$(sh -lc 'echo $PATH')/\$PATH}" > /etc/profile.d/00-restore-env.sh @@ -68,13 +119,24 @@ elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then fi updaterc() { + local _bashrc + local _zshrc if [ "${UPDATE_RC}" = "true" ]; then - echo "Updating /etc/bash.bashrc and /etc/zsh/zshrc..." - if [[ "$(cat /etc/bash.bashrc)" != *"$1"* ]]; then - echo -e "$1" >> /etc/bash.bashrc + case $ADJUSTED_ID in + debian) echo "Updating /etc/bash.bashrc and /etc/zsh/zshrc..." + _bashrc=/etc/bash.bashrc + _zshrc=/etc/zsh/zshrc + ;; + rhel) echo "Updating /etc/bashrc and /etc/zshrc..." + _bashrc=/etc/bashrc + _zshrc=/etc/zshrc + ;; + esac + if [[ "$(cat ${_bashrc})" != *"$1"* ]]; then + echo -e "$1" >> ${_bashrc} fi - if [ -f "/etc/zsh/zshrc" ] && [[ "$(cat /etc/zsh/zshrc)" != *"$1"* ]]; then - echo -e "$1" >> /etc/zsh/zshrc + if [ -f "${_zshrc}" ] && [[ "$(cat ${_zshrc})" != *"$1"* ]]; then + echo -e "$1" >> ${_zshrc} fi fi } @@ -95,15 +157,27 @@ receive_gpg_keys() { export GNUPGHOME="/tmp/tmp-gnupg" mkdir -p ${GNUPGHOME} chmod 700 ${GNUPGHOME} - echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" > ${GNUPGHOME}/dirmngr.conf + # echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" > ${GNUPGHOME}/dirmngr.conf # GPG key download sometimes fails for some reason and retrying fixes it. local retry_count=0 local gpg_ok="false" set +e + num_keys=$(echo "${PYTHON_SOURCE_GPG_KEYS}" | wc -w) until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; do - echo "(*) Downloading GPG key..." - ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys) 2>&1 && gpg_ok="true" + # Because some older oses/base images don't support/have dirmngr we loop over + # keyservers and stop when we've loaded them all + for keyserver in $(echo "${GPG_KEY_SERVERS}" | sed -e 's/keyserver//; /pgp.com/d;') ; do + echo "(*) Downloading GPG keys from ${keyserver} ..." + echo "${keys}" | xargs -n 1 gpg --keyserver ${keyserver} ${keyring_args} --recv-keys 2>&1 + downloaded_keys=$(gpg --fingerprint | grep "fingerprint" | wc -l) + if [ ${num_keys} = ${downloaded_keys} ]; then + gpg_ok="true" + break + fi + done + # echo "(*) Downloading GPG key..." + # ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys) 2>&1 && gpg_ok="true" if [ "${gpg_ok}" != "true" ]; then echo "(*) Failed getting key, retring in 10s..." (( retry_count++ )) @@ -189,20 +263,52 @@ oryx_install() { fi } -apt_get_update() -{ - if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then - echo "Running apt-get update..." - apt-get update -y - fi +pkg_mgr_update() { + case $ADJUSTED_ID in + debian) + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + ${PKG_MGR_CMD} update -y + fi + ;; + rhel) + if [ ${PKG_MGR_CMD} = "microdnf" ]; then + if [ "$(ls /var/cache/yum/* 2>/dev/null | wc -l)" = 0 ]; then + echo "Running ${PKG_MGR_CMD} makecache ..." + ${PKG_MGR_CMD} makecache + fi + else + if [ "$(ls /var/cache/${PKG_MGR_CMD}/* 2>/dev/null | wc -l)" = 0 ]; then + echo "Running ${PKG_MGR_CMD} check-update ..." + set +e + ${PKG_MGR_CMD} check-update + rc=$? + if [ $rc != 0 ] && [ $rc != 100 ]; then + exit 1 + fi + set -e + fi + fi + ;; + esac } # Checks if packages are installed and installs them if not check_packages() { - if ! dpkg -s "$@" > /dev/null 2>&1; then - apt_get_update - apt-get -y install --no-install-recommends "$@" - fi + case ${ADJUSTED_ID} in + debian) + if ! dpkg -s "$@" > /dev/null 2>&1; then + pkg_mgr_update + ${INSTALL_CMD} "$@" + fi + ;; + rhel) + if ! rpm -q "$@" > /dev/null 2>&1; then + pkg_mgr_update + ${INSTALL_CMD} "$@" + fi + ;; + esac } add_symlink() { @@ -218,13 +324,79 @@ add_symlink() { fi } +install_openssl11() { + local _prefix=$1 + mkdir /tmp/openssl11 + ( + cd /tmp/openssl11 + curl -L -f -O https://www.openssl.org/source/openssl-1.1.1w.tar.gz + tar xzf openssl-1.1.1w.tar.gz + cd openssl-1.1.* + ./config --prefix=${_prefix} --openssldir=${_prefix} + make -j $(nproc) + make install_dev + ) + rm -rf /tmp/openssl11 +} + +install_gnupg22() { + local _prefix=$1 + yum-builddep -y gnupg2 + check_packages bzip2 + mkdir -p /tmp/gnupg22 && cd /tmp/gnupg22 + gpg --list-keys + gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 249B39D24F25E3B6 04376F3EE0856959 2071B08A33BD3F06 8A861B1C7EFD60D9 + + curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/libgpg-error/libgpg-error-1.31.tar.gz.sig && \ + curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/libgpg-error/libgpg-error-1.31.tar.gz && \ + curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.8.3.tar.gz && \ + curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.8.3.tar.gz.sig && \ + curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/libassuan/libassuan-2.5.1.tar.bz2 && \ + curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/libassuan/libassuan-2.5.1.tar.bz2.sig && \ + curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/libksba/libksba-1.3.5.tar.bz2 && \ + curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/libksba/libksba-1.3.5.tar.bz2.sig && \ + curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/npth/npth-1.5.tar.bz2 && \ + curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/npth/npth-1.5.tar.bz2.sig && \ + # curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/pinentry/pinentry-1.1.0.tar.bz2 && \ + # curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/pinentry/pinentry-1.1.0.tar.bz2.sig && \ + curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/gnupg/gnupg-2.2.9.tar.bz2 && \ + curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/gnupg/gnupg-2.2.9.tar.bz2.sig && \ + gpg --verify libgpg-error-1.31.tar.gz.sig && tar -xzf libgpg-error-1.31.tar.gz && \ + gpg --verify libgcrypt-1.8.3.tar.gz.sig && tar -xzf libgcrypt-1.8.3.tar.gz && \ + gpg --verify libassuan-2.5.1.tar.bz2.sig && tar -xjf libassuan-2.5.1.tar.bz2 && \ + gpg --verify libksba-1.3.5.tar.bz2.sig && tar -xjf libksba-1.3.5.tar.bz2 && \ + gpg --verify npth-1.5.tar.bz2.sig && tar -xjf npth-1.5.tar.bz2 && \ + # gpg --verify pinentry-1.1.0.tar.bz2.sig && tar -xjf pinentry-1.1.0.tar.bz2 && \ + gpg --verify gnupg-2.2.9.tar.bz2.sig && tar -xjf gnupg-2.2.9.tar.bz2 && \ + cd libgpg-error-1.31/ && ./configure --prefix=${_prefix} && make && make install && cd ../ && \ + cd libgcrypt-1.8.3 && ./configure --prefix=${_prefix} --with-libgpg-error-prefix=${_prefix} && make && make install && cd ../ && \ + cd libassuan-2.5.1 && ./configure --prefix=${_prefix} --with-libgpg-error-prefix=${_prefix} && make && make install && cd ../ && \ + cd libksba-1.3.5 && ./configure --prefix=${_prefix} && make && make install && cd ../ && \ + cd npth-1.5 && ./configure --prefix=${_prefix} && make && make install && cd ../ && \ + # cd pinentry-1.1.0 && ./configure --prefix=${_prefix} --enable-pinentry-curses --disable-pinentry-qt4 && \ + # make && make install && cd ../ && \ + cd gnupg-2.2.9 && LDFLAGS="-Wl,-rpath=${_prefix}/lib" ./configure --prefix=${_prefix} \ + --with-agent-pgm=${_prefix}/bin/gpg-agent \ + --with-dirmngr-pgm=${_prefix}/bin/dirmngr \ + --with-libgpg-error-prefix=${_prefix} \ + --with-libgcrypt-prefix=${_prefix} \ + --with-libassuan-prefix=${_prefix} \ + --with-ksba-prefix=${_prefix} \ + --with-npth-prefix=${_prefix} \ + && make && make install && cd .. + # Without the line below, gpg2 might fail to create / import secret keys !!! + if [ -d ~/.gnugp ]; then rm -ri ~/.gnugp; fi + gpgconf --kill gpg-agent || true + # tidy up + rm -rf cd /tmp/gnupg22 +} + install_from_source() { VERSION=$1 echo "(*) Building Python ${VERSION} from source..." # Install prereqs if missing - check_packages curl ca-certificates gnupg2 tar make gcc libssl-dev zlib1g-dev libncurses5-dev \ - libbz2-dev libreadline-dev libxml2-dev xz-utils libgdbm-dev tk-dev dirmngr \ - libxmlsec1-dev libsqlite3-dev libffi-dev liblzma-dev uuid-dev + check_packages ${REQUIRED_PKGS} + if ! type git > /dev/null 2>&1; then check_packages git fi @@ -239,6 +411,18 @@ install_from_source() { exit 1 fi + # Some platforms/os versions need openssl11 & gpg2.22 installed because they're available + # via common package repositories, for now rhel-7 family, use case statement to + # make it easy to expand + case ${VERSION_CODENAME} in + centos7|rhel7) + install_openssl11 ${INSTALL_PATH} + install_gnupg22 /usr/local/gnupg22 + ADDL_CONFIG_ARGS="--with-openssl=${INSTALL_PATH}" + export LDFLAGS="-Wl,-rpath=${INSTALL_PATH}/lib" + ;; + esac + # Download tgz of source mkdir -p /tmp/python-src ${INSTALL_PATH} cd /tmp/python-src @@ -248,13 +432,19 @@ install_from_source() { curl -sSL -o "/tmp/python-src/${tgz_filename}" "${tgz_url}" # Verify signature + set -x receive_gpg_keys PYTHON_SOURCE_GPG_KEYS + set +x echo "Downloading ${tgz_filename}.asc..." curl -sSL -o "/tmp/python-src/${tgz_filename}.asc" "${tgz_url}.asc" gpg --verify "${tgz_filename}.asc" # Update min protocol for testing only - https://bugs.python.org/issue41561 - cp /etc/ssl/openssl.cnf /tmp/python-src/ + if [ -f /etc/pki/tls/openssl.cnf ]; then + cp /etc/pki/tls/openssl.cnf /tmp/python-src/ + else + cp /etc/ssl/openssl.cnf /tmp/python-src/ + fi sed -i -E 's/MinProtocol[=\ ]+.*/MinProtocol = TLSv1.0/g' /tmp/python-src/openssl.cnf export OPENSSL_CONF=/tmp/python-src/openssl.cnf @@ -262,7 +452,13 @@ install_from_source() { tar -xzf "/tmp/python-src/${tgz_filename}" -C "/tmp/python-src" --strip-components=1 local config_args="" if [ "${OPTIMIZE_BUILD_FROM_SOURCE}" = "true" ]; then - config_args="--enable-optimizations" + config_args="${config_args} --enable-optimizations" + fi + if [ "${ENABLESHARED}" = "true" ]; then + config_args=" ${config_args} --enable-shared" + fi + if [ -n "${ADDL_CONFIG_ARGS}" ]; then + config_args="${config_args} ${ADDL_CONFIG_ARGS}" fi ./configure --prefix="${INSTALL_PATH}" --with-ensurepip=install ${config_args} make -j 8 @@ -355,7 +551,7 @@ install_python() { fi should_install_from_source=false - elif [ "$(dpkg --print-architecture)" = "amd64" ] && [ "${USE_ORYX_IF_AVAILABLE}" = "true" ] && type oryx > /dev/null 2>&1; then + elif [ ${ADJUSTED_ID} = "debian" ] && [ "$(dpkg --print-architecture)" = "amd64" ] && [ "${USE_ORYX_IF_AVAILABLE}" = "true" ] && type oryx > /dev/null 2>&1; then install_using_oryx $version || should_install_from_source=true else should_install_from_source=true @@ -369,9 +565,77 @@ install_python() { export DEBIAN_FRONTEND=noninteractive # General requirements -check_packages curl ca-certificates gnupg2 tar make gcc libssl-dev zlib1g-dev libncurses5-dev \ - libbz2-dev libreadline-dev libxml2-dev xz-utils libgdbm-dev tk-dev dirmngr \ - libxmlsec1-dev libsqlite3-dev libffi-dev liblzma-dev uuid-dev + +REQUIRED_PKGS="" +case ${ADJUSTED_ID} in + debian) + REQUIRED_PKGS="${REQUIRED_PKGS} \ + ca-certificates \ + curl \ + dirmngr \ + gcc \ + gnupg2 \ + libbz2-dev \ + libffi-dev \ + libgdbm-dev \ + liblzma-dev \ + libncurses5-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + libxml2-dev \ + libxmlsec1-dev \ + make \ + tar \ + tk-dev \ + uuid-dev \ + xz-utils \ + zlib1g-dev" + ;; + rhel) + REQUIRED_PKGS="${REQUIRED_PKGS} \ + bzip2-devel \ + curl \ + ca-certificates \ + gcc \ + gdbm-devel \ + gnupg2 \ + libffi-devel \ + libxml2-devel \ + make \ + ncurses-devel \ + openssl-devel \ + readline-devel \ + sqlite-devel \ + tar \ + tk-devel \ + uuid-devel \ + which \ + xmlsec1-devel \ + xz-devel \ + xz \ + zlib-devel" + ;; +esac + +# setup for finding some devel packages: +# rhel9 family +# - dnf install -y 'dnf-command(config-manager)' +# - dnf config-manager --set-enabled crb +# rhel8 family +# - dnf install -y 'dnf-command(config-manager)' +# - dnf config-manager --set-enabled powertools +if [ ${ADJUSTED_ID} = "rhel" ]; then + if [ ${MAJOR_VERSION_ID} = "8" ]; then + ${INSTALL_CMD} 'dnf-command(config-manager)' + dnf config-manager --set-enabled powertools + elif [ ${MAJOR_VERSION_ID} = "9" ]; then + ${INSTALL_CMD} 'dnf-command(config-manager)' + dnf config-manager --set-enabled crb + fi +fi + +check_packages ${REQUIRED_PKGS} # Install Python from source if needed @@ -454,6 +718,14 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ $(python --version) != "" ]]; echo "${util} already installed. Skipping." fi done + # Install modules + if [[ ${#PIP_MODULES[@]} -gt 0 ]]; then + echo "Installing Python modules..." + export PIP_CACHE_DIR=/tmp/pip-tmp/cache + for module in "${PIP_MODULES[@]}"; do + pip3 install "${module}" + done + fi rm -rf /tmp/pip-tmp updaterc "export PIPX_HOME=\"${PIPX_HOME}\"" @@ -492,6 +764,6 @@ if [ "${INSTALL_JUPYTERLAB}" = "true" ]; then fi # Clean up -rm -rf /var/lib/apt/lists/* +clean_up echo "Done!" diff --git a/test/python/alma-8.sh b/test/python/alma-8.sh new file mode 100755 index 000000000..038826d5c --- /dev/null +++ b/test/python/alma-8.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/alma-9.sh b/test/python/alma-9.sh new file mode 100755 index 000000000..038826d5c --- /dev/null +++ b/test/python/alma-9.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/centos-7.sh b/test/python/centos-7.sh new file mode 100755 index 000000000..038826d5c --- /dev/null +++ b/test/python/centos-7.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/scenarios.json b/test/python/scenarios.json index 23b6cfc92..84c8f2f27 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -81,5 +81,29 @@ "version": "3.12" } } + }, + "centos-7": { + "image": "centos:centos7", + "features": { + "python": { + "version": "3.11" + } + } + }, + "alma-8": { + "image": "almalinux:8", + "features": { + "python": { + "version": "latest" + } + } + }, + "alma-9": { + "image": "almalinux:9", + "features": { + "python": { + "version": "latest" + } + } } -} +} \ No newline at end of file From 5e79d618f2926785c4b32437f356aaeada64a753 Mon Sep 17 00:00:00 2001 From: Jeff Putsch Date: Tue, 19 Dec 2023 12:43:23 -0800 Subject: [PATCH 02/15] RHEL support, exisiting tests pass, new RHEL tests pass. --- src/python/NOTES.md | 2 +- src/python/README.md | 4 +- src/python/devcontainer-feature.json | 2 +- src/python/install.sh | 115 +++++++++--------- test/python/alma-8-minimal.sh | 40 ++++++ test/python/alma-9-minimal.sh | 40 ++++++ test/python/alma-9-system.sh | 40 ++++++ test/python/install_alternate_tools.sh | 32 +++++ test/python/install_pip_modules.sh | 19 +++ test/python/install_python_shared_lib_deb.sh | 40 ++++++ test/python/install_python_shared_lib_rhel.sh | 40 ++++++ test/python/scenarios.json | 65 +++++++++- 12 files changed, 376 insertions(+), 63 deletions(-) create mode 100755 test/python/alma-8-minimal.sh create mode 100755 test/python/alma-9-minimal.sh create mode 100755 test/python/alma-9-system.sh create mode 100755 test/python/install_alternate_tools.sh create mode 100755 test/python/install_pip_modules.sh create mode 100755 test/python/install_python_shared_lib_deb.sh create mode 100755 test/python/install_python_shared_lib_rhel.sh diff --git a/src/python/NOTES.md b/src/python/NOTES.md index 19fe92f31..79a308cf5 100644 --- a/src/python/NOTES.md +++ b/src/python/NOTES.md @@ -2,6 +2,6 @@ ## OS Support -This Feature should work on recent versions of Debian/Ubuntu-based distributions with the `apt` package manager installed. +This Feature should work on recent versions of Debian/Ubuntu, RedHat Enterprise Linux, Fedora, Alma, and RockyLinux distributions with the apt, yum, dnf, or microdnf package manager installed. `bash` is required to execute the `install.sh` script. diff --git a/src/python/README.md b/src/python/README.md index 41d29336b..24b1ac94a 100644 --- a/src/python/README.md +++ b/src/python/README.md @@ -18,7 +18,7 @@ Installs the provided version of Python, as well as PIPX, and other common Pytho | version | Select a Python version to install. | string | os-provided | | installTools | Install common Python tools like pylint | boolean | true | | optimize | Optimize Python for performance when compiled (slow) | boolean | false | -| enableShared | Enable building a shared Python library | boolean | true | +| enableShared | Enable building a shared Python library | boolean | false | | installPath | The path where python will be installed. | string | /usr/local/python | | installJupyterlab | Install JupyterLab, a web-based interactive development environment for notebooks | boolean | false | | configureJupyterlabAllowOrigin | Configure JupyterLab to accept HTTP requests from the specified origin | string | - | @@ -35,7 +35,7 @@ Installs the provided version of Python, as well as PIPX, and other common Pytho ## OS Support -This Feature should work on recent versions of Debian/Ubuntu-based distributions with the `apt` package manager installed. +This Feature should work on recent versions of Debian/Ubuntu, RedHat Enterprise Linux, Fedora, Alma, and RockyLinux distributions with the apt, yum, dnf, or microdnf package manager installed. `bash` is required to execute the `install.sh` script. diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index 9b5e5bad9..1c3db85d3 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -44,7 +44,7 @@ }, "enableShared": { "type": "boolean", - "default": true, + "default": false, "description": "Enable building a shared Python library" }, "installPath": { diff --git a/src/python/install.sh b/src/python/install.sh index 9e9a7d509..433dc5cf8 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -10,7 +10,7 @@ PYTHON_VERSION="${VERSION:-"latest"}" # 'system' or 'os-provided' checks the base image first, else installs 'latest' INSTALL_PYTHON_TOOLS="${INSTALLTOOLS:-"true"}" OPTIMIZE_BUILD_FROM_SOURCE="${OPTIMIZE:-"false"}" -ENABLE_SHARED_FROM_SOURCE="${ENABLESHARED:-"true"}" +ENABLE_SHARED_FROM_SOURCE="${ENABLESHARED:-"false"}" PYTHON_INSTALL_PATH="${INSTALLPATH:-"/usr/local/python"}" OVERRIDE_DEFAULT_VERSION="${OVERRIDEDEFAULTVERSION:-"true"}" @@ -27,6 +27,10 @@ CONFIGURE_JUPYTERLAB_ALLOW_ORIGIN="${CONFIGUREJUPYTERLABALLOWORIGIN:-""}" # alongside PYTHON_VERSION, but not set as default. ADDITIONAL_VERSIONS="${ADDITIONALVERSIONS:-""}" +GPG_INSTALL_PATH=/usr/local/gnupg22 +# GPG_CMD will be over-ridden if we need to install gpt2.22 ito GPG_INSTALL_PATH +GPG_CMD=gpg + # Comma-separated list of additional tools to be installed via pipx. IFS="," read -r -a DEFAULT_UTILS <<< "${TOOLSTOINSTALL:-flake8,autopep8,black,yapf,mypy,pydocstyle,pycodestyle,bandit,pipenv,virtualenv,pytest}" @@ -67,19 +71,29 @@ else exit 1 fi +# To find some devel packages, some rhel need to enable specific extra repos +INSTALL_CMD_ADDL_REPO="" +if [ ${ADJUSTED_ID} = "rhel" ]; then + if [ ${MAJOR_VERSION_ID} = "8" ]; then + INSTALL_CMD_ADDL_REPOS="--enablerepo powertools" + elif [ ${MAJOR_VERSION_ID} = "9" ]; then + INSTALL_CMD_ADDL_REPOS="--enablerepo crb" + fi +fi + # Setup INSTALL_CMD & PKG_MGR_CMD if type apt-get > /dev/null 2>&1; then PKG_MGR_CMD=apt-get INSTALL_CMD="${PKG_MGR_CMD} -y install --no-install-recommends" elif type dnf > /dev/null 2>&1; then PKG_MGR_CMD=dnf - INSTALL_CMD="${PKG_MGR_CMD} -y install --refresh --best --nodocs --noplugins --setopt=install_weak_deps=0" + INSTALL_CMD="${PKG_MGR_CMD} ${INSTALL_CMD_ADDL_REPOS} -y install --refresh --best --nodocs --noplugins --setopt=install_weak_deps=0" elif type microdnf > /dev/null 2>&1; then PKG_MGR_CMD=microdnf - INSTALL_CMD="${PKG_MGR_CMD} -y install --refresh --best --nodocs --noplugins --setopt=install_weak_deps=0" + INSTALL_CMD="${PKG_MGR_CMD} ${INSTALL_CMD_ADDL_REPOS} -y install --refresh --best --nodocs --noplugins --setopt=install_weak_deps=0" else PKG_MGR_CMD=yum - INSTALL_CMD="${PKG_MGR_CMD} -y install --noplugins --setopt=install_weak_deps=0" + INSTALL_CMD="${PKG_MGR_CMD} ${INSTALL_CMD_ADDL_REPOS} -y install --noplugins --setopt=install_weak_deps=0" fi # Clean up @@ -90,6 +104,8 @@ clean_up() { ;; rhel) rm -rf /var/cache/dnf/* /var/cache/yum/* + rm -rf /tmp/yum.log + rm -rf ${GPG_INSTALL_PATH} ;; esac } @@ -141,10 +157,12 @@ updaterc() { fi } +# Import the specified key in a variable name passed in as # Import the specified key in a variable name passed in as receive_gpg_keys() { local keys=${!1} local keyring_args="" + local gpg_cmd="gpg" if [ ! -z "$2" ]; then mkdir -p "$(dirname \"$2\")" keyring_args="--no-default-keyring --keyring $2" @@ -157,27 +175,15 @@ receive_gpg_keys() { export GNUPGHOME="/tmp/tmp-gnupg" mkdir -p ${GNUPGHOME} chmod 700 ${GNUPGHOME} - # echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" > ${GNUPGHOME}/dirmngr.conf + echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" > ${GNUPGHOME}/dirmngr.conf # GPG key download sometimes fails for some reason and retrying fixes it. local retry_count=0 local gpg_ok="false" set +e - num_keys=$(echo "${PYTHON_SOURCE_GPG_KEYS}" | wc -w) until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; do - # Because some older oses/base images don't support/have dirmngr we loop over - # keyservers and stop when we've loaded them all - for keyserver in $(echo "${GPG_KEY_SERVERS}" | sed -e 's/keyserver//; /pgp.com/d;') ; do - echo "(*) Downloading GPG keys from ${keyserver} ..." - echo "${keys}" | xargs -n 1 gpg --keyserver ${keyserver} ${keyring_args} --recv-keys 2>&1 - downloaded_keys=$(gpg --fingerprint | grep "fingerprint" | wc -l) - if [ ${num_keys} = ${downloaded_keys} ]; then - gpg_ok="true" - break - fi - done - # echo "(*) Downloading GPG key..." - # ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys) 2>&1 && gpg_ok="true" + echo "(*) Downloading GPG key..." + ( echo "${keys}" | xargs -n 1 ${GPG_CMD} -q ${keyring_args} --recv-keys) 2>&1 && gpg_ok="true" if [ "${gpg_ok}" != "true" ]; then echo "(*) Failed getting key, retring in 10s..." (( retry_count++ )) @@ -394,9 +400,6 @@ install_gnupg22() { install_from_source() { VERSION=$1 echo "(*) Building Python ${VERSION} from source..." - # Install prereqs if missing - check_packages ${REQUIRED_PKGS} - if ! type git > /dev/null 2>&1; then check_packages git fi @@ -416,10 +419,10 @@ install_from_source() { # make it easy to expand case ${VERSION_CODENAME} in centos7|rhel7) + install_gnupg22 ${GPG_INSTALL_PATH} + GPG_CMD=${GPG_INSTALL_PATH}/bin/gpg install_openssl11 ${INSTALL_PATH} - install_gnupg22 /usr/local/gnupg22 - ADDL_CONFIG_ARGS="--with-openssl=${INSTALL_PATH}" - export LDFLAGS="-Wl,-rpath=${INSTALL_PATH}/lib" + ADDL_CONFIG_ARGS="--with-openssl=${INSTALL_PATH} --with-openssl-rpath=${INSTALL_PATH}/lib" ;; esac @@ -432,12 +435,10 @@ install_from_source() { curl -sSL -o "/tmp/python-src/${tgz_filename}" "${tgz_url}" # Verify signature - set -x receive_gpg_keys PYTHON_SOURCE_GPG_KEYS - set +x echo "Downloading ${tgz_filename}.asc..." curl -sSL -o "/tmp/python-src/${tgz_filename}.asc" "${tgz_url}.asc" - gpg --verify "${tgz_filename}.asc" + ${GPG_CMD} --verify "${tgz_filename}.asc" # Update min protocol for testing only - https://bugs.python.org/issue41561 if [ -f /etc/pki/tls/openssl.cnf ]; then @@ -454,8 +455,12 @@ install_from_source() { if [ "${OPTIMIZE_BUILD_FROM_SOURCE}" = "true" ]; then config_args="${config_args} --enable-optimizations" fi + set -x if [ "${ENABLESHARED}" = "true" ]; then config_args=" ${config_args} --enable-shared" + # need double-$: LDFLAGS ends up in Makefile $$ becomes $ when evaluated. + # backslash needed for shell that Make calls escape the $. + export LDFLAGS="${LDFLAGS} -Wl,-rpath="'\$$ORIGIN'"/../lib" fi if [ -n "${ADDL_CONFIG_ARGS}" ]; then config_args="${config_args} ${ADDL_CONFIG_ARGS}" @@ -463,6 +468,7 @@ install_from_source() { ./configure --prefix="${INSTALL_PATH}" --with-ensurepip=install ${config_args} make -j 8 make install + set +x cd /tmp rm -rf /tmp/python-src ${GNUPGHOME} /tmp/vscdc-settings.env @@ -532,7 +538,11 @@ install_python() { version=$1 # If the os-provided versions are "good enough", detect that and bail out. if [ ${version} = "os-provided" ] || [ ${version} = "system" ]; then - check_packages python3 python3-doc python3-pip python3-venv python3-dev python3-tk + if [ ${ADJUSTED_ID} = "debian" ]; then + check_packages python3 python3-doc python3-pip python3-venv python3-dev python3-tk + else + check_packages python3 python3-pip python3-devel python3-tkinter + fi INSTALL_PATH="/usr" local current_bin_path="${CURRENT_PATH}/bin" @@ -557,7 +567,9 @@ install_python() { should_install_from_source=true fi if [ "${should_install_from_source}" = "true" ]; then + set -x install_from_source $version + set +x fi } @@ -595,8 +607,8 @@ case ${ADJUSTED_ID} in rhel) REQUIRED_PKGS="${REQUIRED_PKGS} \ bzip2-devel \ - curl \ ca-certificates \ + findutils \ gcc \ gdbm-devel \ gnupg2 \ @@ -606,6 +618,7 @@ case ${ADJUSTED_ID} in ncurses-devel \ openssl-devel \ readline-devel \ + shadow-utils \ sqlite-devel \ tar \ tk-devel \ @@ -615,29 +628,15 @@ case ${ADJUSTED_ID} in xz-devel \ xz \ zlib-devel" + if ! type curl >/dev/null 2>&1; then + REQUIRED_PKGS="${REQUIRED_PKGS} \ + curl" + fi ;; esac -# setup for finding some devel packages: -# rhel9 family -# - dnf install -y 'dnf-command(config-manager)' -# - dnf config-manager --set-enabled crb -# rhel8 family -# - dnf install -y 'dnf-command(config-manager)' -# - dnf config-manager --set-enabled powertools -if [ ${ADJUSTED_ID} = "rhel" ]; then - if [ ${MAJOR_VERSION_ID} = "8" ]; then - ${INSTALL_CMD} 'dnf-command(config-manager)' - dnf config-manager --set-enabled powertools - elif [ ${MAJOR_VERSION_ID} = "9" ]; then - ${INSTALL_CMD} 'dnf-command(config-manager)' - dnf config-manager --set-enabled crb - fi -fi - check_packages ${REQUIRED_PKGS} - # Install Python from source if needed if [ "${PYTHON_VERSION}" != "none" ]; then if ! cat /etc/group | grep -e "^python:" > /dev/null 2>&1; then @@ -702,7 +701,6 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ $(python --version) != "" ]]; fi # Install tools - echo "Installing Python tools..." export PYTHONUSERBASE=/tmp/pip-tmp export PIP_CACHE_DIR=/tmp/pip-tmp/cache PIPX_DIR="" @@ -718,14 +716,6 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ $(python --version) != "" ]]; echo "${util} already installed. Skipping." fi done - # Install modules - if [[ ${#PIP_MODULES[@]} -gt 0 ]]; then - echo "Installing Python modules..." - export PIP_CACHE_DIR=/tmp/pip-tmp/cache - for module in "${PIP_MODULES[@]}"; do - pip3 install "${module}" - done - fi rm -rf /tmp/pip-tmp updaterc "export PIPX_HOME=\"${PIPX_HOME}\"" @@ -733,6 +723,17 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ $(python --version) != "" ]]; updaterc "if [[ \"\${PATH}\" != *\"\${PIPX_BIN_DIR}\"* ]]; then export PATH=\"\${PATH}:\${PIPX_BIN_DIR}\"; fi" fi +# Install modules via pip +if [[ ${#PIP_MODULES[@]} -gt 0 ]]; then + echo "Installing Python modules..." + export PYTHONUSERBASE=/tmp/pip-tmp + export PIP_CACHE_DIR=/tmp/pip-tmp/cache + for module in "${PIP_MODULES[@]}"; do + pip3 install "${module}" + done + rm -rf /tmp/pip-tmp +fi + # Install JupyterLab if needed if [ "${INSTALL_JUPYTERLAB}" = "true" ]; then if [ -z "${PYTHON_SRC}" ]; then diff --git a/test/python/alma-8-minimal.sh b/test/python/alma-8-minimal.sh new file mode 100755 index 000000000..038826d5c --- /dev/null +++ b/test/python/alma-8-minimal.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/alma-9-minimal.sh b/test/python/alma-9-minimal.sh new file mode 100755 index 000000000..038826d5c --- /dev/null +++ b/test/python/alma-9-minimal.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/alma-9-system.sh b/test/python/alma-9-system.sh new file mode 100755 index 000000000..038826d5c --- /dev/null +++ b/test/python/alma-9-system.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/install_alternate_tools.sh b/test/python/install_alternate_tools.sh new file mode 100755 index 000000000..b98d70997 --- /dev/null +++ b/test/python/install_alternate_tools.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "bandit" bandit --version +check "mypy" mypy --version +check "pipenv" pipenv --version +check "pytest" pytest --version +check "ruff" ruff --version +check "virtualenv" virtualenv --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pipenv" bash -c "which pipenv | grep /usr/local/py-utils/bin/pipenv" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" +check "which ruff" bash -c "which ruff | grep /usr/local/py-utils/bin/ruff" +check "which virtualenv" bash -c "which virtualenv | grep /usr/local/py-utils/bin/virtualenv" + +# Report result +reportResults diff --git a/test/python/install_pip_modules.sh b/test/python/install_pip_modules.sh new file mode 100755 index 000000000..797e651ce --- /dev/null +++ b/test/python/install_pip_modules.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that pip modules are installed +check "click module is installed" bash -c "pip list | grep click" +check "ldap3 module is installed" bash -c "pip list | grep ldap3" + + +# Report result +reportResults diff --git a/test/python/install_python_shared_lib_deb.sh b/test/python/install_python_shared_lib_deb.sh new file mode 100755 index 000000000..038826d5c --- /dev/null +++ b/test/python/install_python_shared_lib_deb.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/install_python_shared_lib_rhel.sh b/test/python/install_python_shared_lib_rhel.sh new file mode 100755 index 000000000..038826d5c --- /dev/null +++ b/test/python/install_python_shared_lib_rhel.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/scenarios.json b/test/python/scenarios.json index 84c8f2f27..e51a3ea46 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -82,6 +82,25 @@ } } }, + "install_alternate_tools": { + "image": "mcr.microsoft.com/devcontainers/base:1-ubuntu-22.04", + "features": { + "python": { + "version": "3.12", + "toolsToInstall": "ruff,mypy,bandit,pipenv,virtualenv,pytest" + } + } + }, + "install_pip_modules": { + "image": "mcr.microsoft.com/devcontainers/base:1-ubuntu-22.04", + "features": { + "python": { + "version": "3.12", + "installTools": false, + "additionalModules": "click,ldap3" + } + } + }, "centos-7": { "image": "centos:centos7", "features": { @@ -94,7 +113,15 @@ "image": "almalinux:8", "features": { "python": { - "version": "latest" + "version": "3.11" + } + } + }, + "alma-8-minimal": { + "image": "almalinux:8-minimal", + "features": { + "python": { + "version": "3.11" } } }, @@ -102,7 +129,41 @@ "image": "almalinux:9", "features": { "python": { - "version": "latest" + "version": "3.11" + } + } + }, + "alma-9-minimal": { + "image": "almalinux:9-minimal", + "features": { + "python": { + "version": "3.11" + } + } + }, + "alma-9-system": { + "image": "almalinux:9", + "features": { + "python": { + "version": "system" + } + } + }, + "install_python_shared_lib_deb": { + "image": "ubuntu:focal", + "features": { + "python": { + "version": "3.11", + "enableShared": true + } + } + }, + "install_python_shared_lib_rhel": { + "image": "almalinux:9", + "features": { + "python": { + "version": "3.11", + "enableShared": true } } } From 4f77b37a1178e89095c9a2e232f889d0d6e99e95 Mon Sep 17 00:00:00 2001 From: Jeff Putsch Date: Tue, 30 Jan 2024 12:25:58 -0800 Subject: [PATCH 03/15] add tests, cleanup install organization --- src/python/README.md | 3 +- src/python/devcontainer-feature.json | 2 +- src/python/install.sh | 59 ++++++++++++--------- test/python/{alma-9-system.sh => fedora.sh} | 0 test/python/rocky-8-minimal.sh | 40 ++++++++++++++ test/python/rocky-8.sh | 40 ++++++++++++++ test/python/rocky-9-minimal.sh | 40 ++++++++++++++ test/python/rocky-9.sh | 40 ++++++++++++++ test/python/scenarios.json | 38 +++++++++++-- 9 files changed, 231 insertions(+), 31 deletions(-) rename test/python/{alma-9-system.sh => fedora.sh} (100%) create mode 100755 test/python/rocky-8-minimal.sh create mode 100755 test/python/rocky-8.sh create mode 100755 test/python/rocky-9-minimal.sh create mode 100755 test/python/rocky-9.sh diff --git a/src/python/README.md b/src/python/README.md index 24b1ac94a..2dd092ae7 100644 --- a/src/python/README.md +++ b/src/python/README.md @@ -18,7 +18,6 @@ Installs the provided version of Python, as well as PIPX, and other common Pytho | version | Select a Python version to install. | string | os-provided | | installTools | Install common Python tools like pylint | boolean | true | | optimize | Optimize Python for performance when compiled (slow) | boolean | false | -| enableShared | Enable building a shared Python library | boolean | false | | installPath | The path where python will be installed. | string | /usr/local/python | | installJupyterlab | Install JupyterLab, a web-based interactive development environment for notebooks | boolean | false | | configureJupyterlabAllowOrigin | Configure JupyterLab to accept HTTP requests from the specified origin | string | - | @@ -35,7 +34,7 @@ Installs the provided version of Python, as well as PIPX, and other common Pytho ## OS Support -This Feature should work on recent versions of Debian/Ubuntu, RedHat Enterprise Linux, Fedora, Alma, and RockyLinux distributions with the apt, yum, dnf, or microdnf package manager installed. +This Feature should work on recent versions of Debian/Ubuntu-based distributions with the `apt` package manager installed. `bash` is required to execute the `install.sh` script. diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index 1c3db85d3..d0c97739b 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "python", - "version": "1.3.1", + "version": "1.4.0", "name": "Python", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/python", "description": "Installs the provided version of Python, as well as PIPX, and other common Python utilities. JupyterLab is conditionally installed with the python feature. Note: May require source code compilation.", diff --git a/src/python/install.sh b/src/python/install.sh index 433dc5cf8..8090c0cc0 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -85,12 +85,12 @@ fi if type apt-get > /dev/null 2>&1; then PKG_MGR_CMD=apt-get INSTALL_CMD="${PKG_MGR_CMD} -y install --no-install-recommends" -elif type dnf > /dev/null 2>&1; then - PKG_MGR_CMD=dnf - INSTALL_CMD="${PKG_MGR_CMD} ${INSTALL_CMD_ADDL_REPOS} -y install --refresh --best --nodocs --noplugins --setopt=install_weak_deps=0" elif type microdnf > /dev/null 2>&1; then PKG_MGR_CMD=microdnf INSTALL_CMD="${PKG_MGR_CMD} ${INSTALL_CMD_ADDL_REPOS} -y install --refresh --best --nodocs --noplugins --setopt=install_weak_deps=0" +elif type dnf > /dev/null 2>&1; then + PKG_MGR_CMD=dnf + INSTALL_CMD="${PKG_MGR_CMD} ${INSTALL_CMD_ADDL_REPOS} -y install --refresh --best --nodocs --noplugins --setopt=install_weak_deps=0" else PKG_MGR_CMD=yum INSTALL_CMD="${PKG_MGR_CMD} ${INSTALL_CMD_ADDL_REPOS} -y install --noplugins --setopt=install_weak_deps=0" @@ -112,27 +112,6 @@ clean_up() { clean_up -# Ensure that login shells get the correct path if the user updated the PATH using ENV. -rm -f /etc/profile.d/00-restore-env.sh -echo "export PATH=${PATH//$(sh -lc 'echo $PATH')/\$PATH}" > /etc/profile.d/00-restore-env.sh -chmod +x /etc/profile.d/00-restore-env.sh - -# Determine the appropriate non-root user -if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then - USERNAME="" - POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") - for CURRENT_USER in "${POSSIBLE_USERS[@]}"; do - if id -u ${CURRENT_USER} > /dev/null 2>&1; then - USERNAME=${CURRENT_USER} - break - fi - done - if [ "${USERNAME}" = "" ]; then - USERNAME=root - fi -elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then - USERNAME=root -fi updaterc() { local _bashrc @@ -573,6 +552,33 @@ install_python() { fi } +# Ensure that login shells get the correct path if the user updated the PATH using ENV. +rm -f /etc/profile.d/00-restore-env.sh +echo "export PATH=${PATH//$(sh -lc 'echo $PATH')/\$PATH}" > /etc/profile.d/00-restore-env.sh +chmod +x /etc/profile.d/00-restore-env.sh + +# Some distributions do not install awk by default (e.g. Mariner) +if ! type awk >/dev/null 2>&1; then + check_packages awk +fi + +# Determine the appropriate non-root user +if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then + USERNAME="" + POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") + for CURRENT_USER in "${POSSIBLE_USERS[@]}"; do + if id -u ${CURRENT_USER} > /dev/null 2>&1; then + USERNAME=${CURRENT_USER} + break + fi + done + if [ "${USERNAME}" = "" ]; then + USERNAME=root + fi +elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then + USERNAME=root +fi + # Ensure apt is in non-interactive to avoid prompts export DEBIAN_FRONTEND=noninteractive @@ -621,7 +627,6 @@ case ${ADJUSTED_ID} in shadow-utils \ sqlite-devel \ tar \ - tk-devel \ uuid-devel \ which \ xmlsec1-devel \ @@ -632,6 +637,10 @@ case ${ADJUSTED_ID} in REQUIRED_PKGS="${REQUIRED_PKGS} \ curl" fi + if [ "${ID}" != "mariner" ]; then + REQUIRED_PKGS="${REQUIRED_PKGS} \ + tk-devel" + fi ;; esac diff --git a/test/python/alma-9-system.sh b/test/python/fedora.sh similarity index 100% rename from test/python/alma-9-system.sh rename to test/python/fedora.sh diff --git a/test/python/rocky-8-minimal.sh b/test/python/rocky-8-minimal.sh new file mode 100755 index 000000000..038826d5c --- /dev/null +++ b/test/python/rocky-8-minimal.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/rocky-8.sh b/test/python/rocky-8.sh new file mode 100755 index 000000000..038826d5c --- /dev/null +++ b/test/python/rocky-8.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/rocky-9-minimal.sh b/test/python/rocky-9-minimal.sh new file mode 100755 index 000000000..038826d5c --- /dev/null +++ b/test/python/rocky-9-minimal.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/rocky-9.sh b/test/python/rocky-9.sh new file mode 100755 index 000000000..038826d5c --- /dev/null +++ b/test/python/rocky-9.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/scenarios.json b/test/python/scenarios.json index e51a3ea46..d6a89cbbc 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -141,11 +141,43 @@ } } }, - "alma-9-system": { - "image": "almalinux:9", + "rocky-8": { + "image": "rockylinux:8", + "features": { + "python": { + "version": "3.11" + } + } + }, + "rocky-8-minimal": { + "image": "rockylinux:8-minimal", + "features": { + "python": { + "version": "3.11" + } + } + }, + "rocky-9": { + "image": "rockylinux:9", "features": { "python": { - "version": "system" + "version": "3.11" + } + } + }, + "rocky-9-minimal": { + "image": "rockylinux:9-minimal", + "features": { + "python": { + "version": "3.11" + } + } + }, + "fedora": { + "image": "fedora", + "features": { + "python": { + "version": "3.11" } } }, From a1fd5b2a140839b16c703af00eb2610ae98411de Mon Sep 17 00:00:00 2001 From: Jeff Putsch Date: Wed, 31 Jan 2024 09:24:01 -0800 Subject: [PATCH 04/15] update testing to include RHEL tests --- ...stall_additional_jupyterlab_rhel_family.sh | 26 +++++++ .../install_additional_python_rhel_family.sh | 41 +++++++++++ .../install_alternate_tools_rhel_family.sh | 32 ++++++++ test/python/install_jupyterlab_rhel_family.sh | 26 +++++++ .../install_os_provided_python_mariner.sh | 40 ++++++++++ .../install_os_provided_python_rhel_family.sh | 40 ++++++++++ .../python/install_pip_modules_rhel_family.sh | 19 +++++ ...b_rhel.sh => install_python_shared_lib.sh} | 0 .../install_python_shared_lib_rhel_family.sh | 40 ++++++++++ test/python/scenarios.json | 73 ++++++++++++++++++- 10 files changed, 335 insertions(+), 2 deletions(-) create mode 100644 test/python/install_additional_jupyterlab_rhel_family.sh create mode 100644 test/python/install_additional_python_rhel_family.sh create mode 100644 test/python/install_alternate_tools_rhel_family.sh create mode 100644 test/python/install_jupyterlab_rhel_family.sh create mode 100755 test/python/install_os_provided_python_mariner.sh create mode 100644 test/python/install_os_provided_python_rhel_family.sh create mode 100755 test/python/install_pip_modules_rhel_family.sh rename test/python/{install_python_shared_lib_rhel.sh => install_python_shared_lib.sh} (100%) create mode 100755 test/python/install_python_shared_lib_rhel_family.sh diff --git a/test/python/install_additional_jupyterlab_rhel_family.sh b/test/python/install_additional_jupyterlab_rhel_family.sh new file mode 100644 index 000000000..58c4f7f54 --- /dev/null +++ b/test/python/install_additional_jupyterlab_rhel_family.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Always run these checks as the non-root user +user="$(whoami)" +check "user" grep vscode <<< "$user" + +# Check for an installation of JupyterLab +check "version" jupyter lab --version + +# Check location of JupyterLab installation +packages="$(python3 -m pip list)" +check "location" grep jupyter <<< "$packages" + +# Check for git extension +check "jupyterlab_git" grep jupyterlab_git <<< "$packages" + +# Check for correct JupyterLab configuration +check "config" grep ".*.allow_origin = '*'" /home/vscode/.jupyter/jupyter_server_config.py + +# Report result +reportResults diff --git a/test/python/install_additional_python_rhel_family.sh b/test/python/install_additional_python_rhel_family.sh new file mode 100644 index 000000000..d584f4be9 --- /dev/null +++ b/test/python/install_additional_python_rhel_family.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +check "python version 3.11 installed as default" bash -c "python --version | grep 3.11" +check "python3 version 3.11 installed as default" bash -c "python3 --version | grep 3.11" +check "python version 3.10.5 installed" bash -c "ls -l /usr/local/python | grep 3.10.5" +check "python version 3.8 installed" bash -c "ls -l /usr/local/python | grep 3.8" +check "python version 3.9.13 installed" bash -c "ls -l /usr/local/python | grep 3.9.13" + +# Check that tools can execute - make sure something didn't get messed up in this scenario +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/install_alternate_tools_rhel_family.sh b/test/python/install_alternate_tools_rhel_family.sh new file mode 100644 index 000000000..b98d70997 --- /dev/null +++ b/test/python/install_alternate_tools_rhel_family.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "bandit" bandit --version +check "mypy" mypy --version +check "pipenv" pipenv --version +check "pytest" pytest --version +check "ruff" ruff --version +check "virtualenv" virtualenv --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pipenv" bash -c "which pipenv | grep /usr/local/py-utils/bin/pipenv" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" +check "which ruff" bash -c "which ruff | grep /usr/local/py-utils/bin/ruff" +check "which virtualenv" bash -c "which virtualenv | grep /usr/local/py-utils/bin/virtualenv" + +# Report result +reportResults diff --git a/test/python/install_jupyterlab_rhel_family.sh b/test/python/install_jupyterlab_rhel_family.sh new file mode 100644 index 000000000..58c4f7f54 --- /dev/null +++ b/test/python/install_jupyterlab_rhel_family.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Always run these checks as the non-root user +user="$(whoami)" +check "user" grep vscode <<< "$user" + +# Check for an installation of JupyterLab +check "version" jupyter lab --version + +# Check location of JupyterLab installation +packages="$(python3 -m pip list)" +check "location" grep jupyter <<< "$packages" + +# Check for git extension +check "jupyterlab_git" grep jupyterlab_git <<< "$packages" + +# Check for correct JupyterLab configuration +check "config" grep ".*.allow_origin = '*'" /home/vscode/.jupyter/jupyter_server_config.py + +# Report result +reportResults diff --git a/test/python/install_os_provided_python_mariner.sh b/test/python/install_os_provided_python_mariner.sh new file mode 100755 index 000000000..e39b51412 --- /dev/null +++ b/test/python/install_os_provided_python_mariner.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +check "python3 is installed" python3 --version +check "python is installed" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/install_os_provided_python_rhel_family.sh b/test/python/install_os_provided_python_rhel_family.sh new file mode 100644 index 000000000..e39b51412 --- /dev/null +++ b/test/python/install_os_provided_python_rhel_family.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +check "python3 is installed" python3 --version +check "python is installed" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/install_pip_modules_rhel_family.sh b/test/python/install_pip_modules_rhel_family.sh new file mode 100755 index 000000000..797e651ce --- /dev/null +++ b/test/python/install_pip_modules_rhel_family.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that pip modules are installed +check "click module is installed" bash -c "pip list | grep click" +check "ldap3 module is installed" bash -c "pip list | grep ldap3" + + +# Report result +reportResults diff --git a/test/python/install_python_shared_lib_rhel.sh b/test/python/install_python_shared_lib.sh similarity index 100% rename from test/python/install_python_shared_lib_rhel.sh rename to test/python/install_python_shared_lib.sh diff --git a/test/python/install_python_shared_lib_rhel_family.sh b/test/python/install_python_shared_lib_rhel_family.sh new file mode 100755 index 000000000..038826d5c --- /dev/null +++ b/test/python/install_python_shared_lib_rhel_family.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests +check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "current symlink works" /usr/local/python/current/bin/python --version +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +# Report result +reportResults diff --git a/test/python/scenarios.json b/test/python/scenarios.json index d6a89cbbc..a8adc0fe6 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -8,6 +8,15 @@ } } }, + "install_additional_python_rhel_family": { + "image": "almalinux:8", + "features": { + "python": { + "version": "3.11", + "additionalVersions": "3.8,3.9.13,3.10.5" + } + } + }, "install_jupyterlab": { "image": "mcr.microsoft.com/devcontainers/base:focal", "remoteUser": "vscode", @@ -19,6 +28,20 @@ } } }, + "install_jupyterlab_rhel_family": { + "image": "almalinux:8", + "remoteUser": "vscode", + "features": { + "common-utils": { + "username": "vscode" + }, + "python": { + "version": "latest", + "installJupyterlab": true, + "configureJupyterlabAllowOrigin": "*" + } + } + }, "install_additional_jupyterlab": { "image": "mcr.microsoft.com/devcontainers/base:focal", "remoteUser": "vscode", @@ -31,12 +54,39 @@ } } }, + "install_additional_jupyterlab_rhel_family": { + "image": "almalinux:8", + "remoteUser": "vscode", + "features": { + "common-utils": { + "username": "vscode" + }, + "python": { + "version": "latest", + "additionalVersions": "3.9", + "installJupyterlab": true, + "configureJupyterlabAllowOrigin": "*" + } + } + }, "install_os_provided_python": { "image": "mcr.microsoft.com/devcontainers/base:1-bullseye", "features": { "python": "os-provided" } }, + "install_os_provided_python_rhel_family": { + "image": "almalinux:8", + "features": { + "python": "os-provided" + } + }, + "install_os_provided_python_mariner": { + "image": "mcr.microsoft.com/cbl-mariner/base/core:2.0", + "features": { + "python": "os-provided" + } + }, "install_jupyterlab_from_python_image": { "image": "mcr.microsoft.com/devcontainers/python:3.10-bullseye", "remoteUser": "vscode", @@ -91,6 +141,15 @@ } } }, + "install_alternate_tools_rhel_family": { + "image": "almalinux:8", + "features": { + "python": { + "version": "3.12", + "toolsToInstall": "ruff,mypy,bandit,pipenv,virtualenv,pytest" + } + } + }, "install_pip_modules": { "image": "mcr.microsoft.com/devcontainers/base:1-ubuntu-22.04", "features": { @@ -101,6 +160,16 @@ } } }, + "install_pip_modules_rhel_family": { + "image": "almalinux:8", + "features": { + "python": { + "version": "3.12", + "installTools": false, + "additionalModules": "click,ldap3" + } + } + }, "centos-7": { "image": "centos:centos7", "features": { @@ -190,8 +259,8 @@ } } }, - "install_python_shared_lib_rhel": { - "image": "almalinux:9", + "install_python_shared_lib_rhel_family": { + "image": "almalinux:8", "features": { "python": { "version": "3.11", From 6fc48e392bee21c94a0fabfdaadce2e58836c072 Mon Sep 17 00:00:00 2001 From: Jeff Putsch Date: Wed, 31 Jan 2024 09:24:34 -0800 Subject: [PATCH 05/15] update testing to include RHEL tests --- src/python/install.sh | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/python/install.sh b/src/python/install.sh index 8090c0cc0..3534d5c0e 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -342,8 +342,6 @@ install_gnupg22() { curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/libksba/libksba-1.3.5.tar.bz2.sig && \ curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/npth/npth-1.5.tar.bz2 && \ curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/npth/npth-1.5.tar.bz2.sig && \ - # curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/pinentry/pinentry-1.1.0.tar.bz2 && \ - # curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/pinentry/pinentry-1.1.0.tar.bz2.sig && \ curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/gnupg/gnupg-2.2.9.tar.bz2 && \ curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/gnupg/gnupg-2.2.9.tar.bz2.sig && \ gpg --verify libgpg-error-1.31.tar.gz.sig && tar -xzf libgpg-error-1.31.tar.gz && \ @@ -351,15 +349,12 @@ install_gnupg22() { gpg --verify libassuan-2.5.1.tar.bz2.sig && tar -xjf libassuan-2.5.1.tar.bz2 && \ gpg --verify libksba-1.3.5.tar.bz2.sig && tar -xjf libksba-1.3.5.tar.bz2 && \ gpg --verify npth-1.5.tar.bz2.sig && tar -xjf npth-1.5.tar.bz2 && \ - # gpg --verify pinentry-1.1.0.tar.bz2.sig && tar -xjf pinentry-1.1.0.tar.bz2 && \ gpg --verify gnupg-2.2.9.tar.bz2.sig && tar -xjf gnupg-2.2.9.tar.bz2 && \ cd libgpg-error-1.31/ && ./configure --prefix=${_prefix} && make && make install && cd ../ && \ cd libgcrypt-1.8.3 && ./configure --prefix=${_prefix} --with-libgpg-error-prefix=${_prefix} && make && make install && cd ../ && \ cd libassuan-2.5.1 && ./configure --prefix=${_prefix} --with-libgpg-error-prefix=${_prefix} && make && make install && cd ../ && \ cd libksba-1.3.5 && ./configure --prefix=${_prefix} && make && make install && cd ../ && \ cd npth-1.5 && ./configure --prefix=${_prefix} && make && make install && cd ../ && \ - # cd pinentry-1.1.0 && ./configure --prefix=${_prefix} --enable-pinentry-curses --disable-pinentry-qt4 && \ - # make && make install && cd ../ && \ cd gnupg-2.2.9 && LDFLAGS="-Wl,-rpath=${_prefix}/lib" ./configure --prefix=${_prefix} \ --with-agent-pgm=${_prefix}/bin/gpg-agent \ --with-dirmngr-pgm=${_prefix}/bin/dirmngr \ @@ -627,6 +622,7 @@ case ${ADJUSTED_ID} in shadow-utils \ sqlite-devel \ tar \ + tk-devel \ uuid-devel \ which \ xmlsec1-devel \ @@ -637,10 +633,6 @@ case ${ADJUSTED_ID} in REQUIRED_PKGS="${REQUIRED_PKGS} \ curl" fi - if [ "${ID}" != "mariner" ]; then - REQUIRED_PKGS="${REQUIRED_PKGS} \ - tk-devel" - fi ;; esac From 11414e26ca7c918a1088c7cee1a374a6849c8354 Mon Sep 17 00:00:00 2001 From: Jeff Putsch Date: Wed, 31 Jan 2024 09:45:50 -0800 Subject: [PATCH 06/15] undo addition of installing additional pip modules --- src/python/devcontainer-feature.json | 5 ----- src/python/install.sh | 14 ------------- test/python/install_pip_modules.sh | 19 ------------------ .../python/install_pip_modules_rhel_family.sh | 19 ------------------ test/python/scenarios.json | 20 ------------------- 5 files changed, 77 deletions(-) delete mode 100755 test/python/install_pip_modules.sh delete mode 100755 test/python/install_pip_modules_rhel_family.sh diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index d0c97739b..74ac1b676 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -32,11 +32,6 @@ "default": "flake8,autopep8,black,yapf,mypy,pydocstyle,pycodestyle,bandit,pipenv,virtualenv,pytest,pylint", "description": "Comma-separated list of tools to install when installTools is true" }, - "additionalModules": { - "type": "string", - "default": "", - "description": "Comma-separated list of modules to install via PIP (not pipx)" - }, "optimize": { "type": "boolean", "default": false, diff --git a/src/python/install.sh b/src/python/install.sh index 3534d5c0e..4c258d90b 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -34,9 +34,6 @@ GPG_CMD=gpg # Comma-separated list of additional tools to be installed via pipx. IFS="," read -r -a DEFAULT_UTILS <<< "${TOOLSTOINSTALL:-flake8,autopep8,black,yapf,mypy,pydocstyle,pycodestyle,bandit,pipenv,virtualenv,pytest}" -# Comma-separated list of additional pyhton modules to install via pip. -IFS="," read -r -a PIP_MODULES <<< "${ADDITIONALMODULES:-}" - PYTHON_SOURCE_GPG_KEYS="64E628F8D684696D B26995E310250568 2D347EA6AA65421D FB9921286F5E1540 3A5CA953F73C700D 04C367C218ADD4FF 0EDDC5F26A45C816 6AF053F07D9DC8D2 C9BE28DEE6DF025C 126EB563A74B06BF D9866941EA5BBD71 ED9D77D5 A821E680E5FA6305" GPG_KEY_SERVERS="keyserver hkp://keyserver.ubuntu.com @@ -724,17 +721,6 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ $(python --version) != "" ]]; updaterc "if [[ \"\${PATH}\" != *\"\${PIPX_BIN_DIR}\"* ]]; then export PATH=\"\${PATH}:\${PIPX_BIN_DIR}\"; fi" fi -# Install modules via pip -if [[ ${#PIP_MODULES[@]} -gt 0 ]]; then - echo "Installing Python modules..." - export PYTHONUSERBASE=/tmp/pip-tmp - export PIP_CACHE_DIR=/tmp/pip-tmp/cache - for module in "${PIP_MODULES[@]}"; do - pip3 install "${module}" - done - rm -rf /tmp/pip-tmp -fi - # Install JupyterLab if needed if [ "${INSTALL_JUPYTERLAB}" = "true" ]; then if [ -z "${PYTHON_SRC}" ]; then diff --git a/test/python/install_pip_modules.sh b/test/python/install_pip_modules.sh deleted file mode 100755 index 797e651ce..000000000 --- a/test/python/install_pip_modules.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -set -e - -# Optional: Import test library -source dev-container-features-test-lib - -# Definition specific tests -check "version" python --version -check "pip is installed" pip --version -check "pip is installed" pip3 --version - -# Check that pip modules are installed -check "click module is installed" bash -c "pip list | grep click" -check "ldap3 module is installed" bash -c "pip list | grep ldap3" - - -# Report result -reportResults diff --git a/test/python/install_pip_modules_rhel_family.sh b/test/python/install_pip_modules_rhel_family.sh deleted file mode 100755 index 797e651ce..000000000 --- a/test/python/install_pip_modules_rhel_family.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -set -e - -# Optional: Import test library -source dev-container-features-test-lib - -# Definition specific tests -check "version" python --version -check "pip is installed" pip --version -check "pip is installed" pip3 --version - -# Check that pip modules are installed -check "click module is installed" bash -c "pip list | grep click" -check "ldap3 module is installed" bash -c "pip list | grep ldap3" - - -# Report result -reportResults diff --git a/test/python/scenarios.json b/test/python/scenarios.json index a8adc0fe6..a377dae1a 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -150,26 +150,6 @@ } } }, - "install_pip_modules": { - "image": "mcr.microsoft.com/devcontainers/base:1-ubuntu-22.04", - "features": { - "python": { - "version": "3.12", - "installTools": false, - "additionalModules": "click,ldap3" - } - } - }, - "install_pip_modules_rhel_family": { - "image": "almalinux:8", - "features": { - "python": { - "version": "3.12", - "installTools": false, - "additionalModules": "click,ldap3" - } - } - }, "centos-7": { "image": "centos:centos7", "features": { From 4007701ab9e9eae7152c8c0ff3ef403ff84df124 Mon Sep 17 00:00:00 2001 From: Jeff Putsch Date: Thu, 1 Feb 2024 20:55:07 -0800 Subject: [PATCH 07/15] fix errors installing os-provided Python on recent Debian systems and on Mariner systems --- src/python/install.sh | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/python/install.sh b/src/python/install.sh index 4c258d90b..8bc473da4 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -512,7 +512,11 @@ install_python() { if [ ${ADJUSTED_ID} = "debian" ]; then check_packages python3 python3-doc python3-pip python3-venv python3-dev python3-tk else - check_packages python3 python3-pip python3-devel python3-tkinter + if [ ${ID} != "mariner" ]; then + check_packages python3 python3-pip python3-devel python3-tkinter + else + check_packages python3 python3-pip python3-devel + fi fi INSTALL_PATH="/usr" @@ -619,7 +623,6 @@ case ${ADJUSTED_ID} in shadow-utils \ sqlite-devel \ tar \ - tk-devel \ uuid-devel \ which \ xmlsec1-devel \ @@ -630,6 +633,11 @@ case ${ADJUSTED_ID} in REQUIRED_PKGS="${REQUIRED_PKGS} \ curl" fi + # Mariner does not have tk-devel package available + if [ ${ID} != "mariner" ]; then + REQUIRED_PKGS="${REQUIRED_PKGS} \ + tk-devel" + fi ;; esac @@ -693,7 +701,7 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ $(python --version) != "" ]]; find "${PIPX_HOME}" -type d -print0 | xargs -0 -n 1 chmod g+s # Update pip if not using os provided python - if [[ $(python --version) != "" ]] || [[ ${PYTHON_VERSION} != "os-provided" ]] && [[ ${PYTHON_VERSION} != "system" ]] && [[ ${PYTHON_VERSION} != "none" ]]; then + if [[ $(python --version 2>/dev/null) != "" ]] && [[ ${PYTHON_VERSION} != "os-provided" ]] && [[ ${PYTHON_VERSION} != "system" ]] && [[ ${PYTHON_VERSION} != "none" ]]; then echo "Updating pip..." python -m pip install --no-cache-dir --upgrade pip fi @@ -703,7 +711,15 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ $(python --version) != "" ]]; export PIP_CACHE_DIR=/tmp/pip-tmp/cache PIPX_DIR="" if ! type pipx > /dev/null 2>&1; then - pip3 install --disable-pip-version-check --no-cache-dir --user pipx 2>&1 + break_system_packages="" + if [ ${ADJUSTED_ID} = "debian" ] && [ ${MAJOR_VERSION_ID} -ge 12 ]; then + # Debian versions >= 12 require that we use pip with the "--break-system-packages" + # option to avoid errors... This does not really breack system packages due + # to the setting of PYTHONUSERBASE above, but does get us past Debian checks for + # installing python packages into the system python install. + break_system_packages="--break-system-packages" + fi + pip3 install ${break_system_packages} --disable-pip-version-check --no-cache-dir --user pipx 2>&1 /tmp/pip-tmp/bin/pipx install --pip-args=--no-cache-dir pipx PIPX_DIR="/tmp/pip-tmp/bin/" fi From f20275f03da2a5cb6dd61c8ad83cdacb38e8e8a1 Mon Sep 17 00:00:00 2001 From: Jeff Putsch Date: Thu, 1 Feb 2024 21:29:00 -0800 Subject: [PATCH 08/15] adjust to properly use newly installed python (PYTHON_SRC) instead of assuming "python" will work --- src/python/install.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/python/install.sh b/src/python/install.sh index 8bc473da4..fe0af4e3a 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -542,9 +542,7 @@ install_python() { should_install_from_source=true fi if [ "${should_install_from_source}" = "true" ]; then - set -x install_from_source $version - set +x fi } @@ -684,7 +682,7 @@ else fi # Install Python tools if needed -if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ $(python --version) != "" ]]; then +if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ -n "${PYTHON_SRC}" ]]; then echo 'Installing Python tools...' export PIPX_BIN_DIR="${PIPX_HOME}/bin" PATH="${PATH}:${PIPX_BIN_DIR}" @@ -701,9 +699,9 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ $(python --version) != "" ]]; find "${PIPX_HOME}" -type d -print0 | xargs -0 -n 1 chmod g+s # Update pip if not using os provided python - if [[ $(python --version 2>/dev/null) != "" ]] && [[ ${PYTHON_VERSION} != "os-provided" ]] && [[ ${PYTHON_VERSION} != "system" ]] && [[ ${PYTHON_VERSION} != "none" ]]; then + if [[ -n "${PYTHON_SRC}" ]] && [[ ${PYTHON_VERSION} != "os-provided" ]] && [[ ${PYTHON_VERSION} != "system" ]] && [[ ${PYTHON_VERSION} != "none" ]]; then echo "Updating pip..." - python -m pip install --no-cache-dir --upgrade pip + ${PYTHON_SRC} -m pip install --no-cache-dir --upgrade pip fi # Install tools From dad4f8dac7d16cc2d94c8efdaadff5c22073ec8c Mon Sep 17 00:00:00 2001 From: Jeff Putsch Date: Fri, 2 Feb 2024 13:23:25 -0800 Subject: [PATCH 09/15] When installing pipx, check if python is marked as externally managed. If so, add "--break-system-packages" to the pip install flags. This does not really breack system packages due to the setting of PYTHONUSERBASE during the install of pipx, but does get us past checks for installing python packages into the system python install. --- src/python/install.sh | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/python/install.sh b/src/python/install.sh index fe0af4e3a..a53da7f61 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -546,6 +546,21 @@ install_python() { fi } +python_is_externally_managed() { + local _python_cmd=$1 + local python_stdlib_dir=$( + ${_python_cmd} -c ' +import sys +import sysconfig +sys.prefix == sys.base_prefix and print(sysconfig.get_path("stdlib", sysconfig.get_default_scheme()))' + ) + if [ -f ${python_stdlib_dir}/EXTERNALLY-MANAGED ]; then + return 0 + else + return 1 + fi +} + # Ensure that login shells get the correct path if the user updated the PATH using ENV. rm -f /etc/profile.d/00-restore-env.sh echo "export PATH=${PATH//$(sh -lc 'echo $PATH')/\$PATH}" > /etc/profile.d/00-restore-env.sh @@ -709,12 +724,13 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ -n "${PYTHON_SRC}" ]]; then export PIP_CACHE_DIR=/tmp/pip-tmp/cache PIPX_DIR="" if ! type pipx > /dev/null 2>&1; then - break_system_packages="" - if [ ${ADJUSTED_ID} = "debian" ] && [ ${MAJOR_VERSION_ID} -ge 12 ]; then - # Debian versions >= 12 require that we use pip with the "--break-system-packages" - # option to avoid errors... This does not really breack system packages due - # to the setting of PYTHONUSERBASE above, but does get us past Debian checks for - # installing python packages into the system python install. + + # python install is marked as externally managed, we need to pass + # pip a "--break-system-packages" option to avoid errors... This does not + # really breack system packages dueto the setting of PYTHONUSERBASE above, + # but does get us past checks for installing python packages into the + # system python install. + if python_is_externally_managed ${PYTHON_SRC}; then break_system_packages="--break-system-packages" fi pip3 install ${break_system_packages} --disable-pip-version-check --no-cache-dir --user pipx 2>&1 From 50c7d12c9e8d6bf5c1c7704f597e9da7efd6454c Mon Sep 17 00:00:00 2001 From: Jeff Putsch Date: Fri, 2 Feb 2024 13:47:41 -0800 Subject: [PATCH 10/15] merge from main --- src/python/install.sh | 10 +++ ...tall_python310_setuptools_vulnerability.sh | 89 ++++++++++++++++++ ...tall_python311_setuptools_vulnerability.sh | 90 +++++++++++++++++++ test/python/scenarios.json | 18 ++++ 4 files changed, 207 insertions(+) create mode 100644 test/python/install_python310_setuptools_vulnerability.sh create mode 100644 test/python/install_python311_setuptools_vulnerability.sh diff --git a/src/python/install.sh b/src/python/install.sh index a53da7f61..a2ec5228f 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -744,6 +744,16 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ -n "${PYTHON_SRC}" ]]; then echo "${util} already installed. Skipping." fi done + + # Temporary: Removes “setup tools” metadata directory due to https://github.com/advisories/GHSA-r9hx-vwmv-q579 + + VULNERABLE_VERSIONS=("3.10" "3.11") + RUN_TIME_PY_VER_DETECT=$(python --version 2>&1) + PY_MAJOR_MINOR_VER=${RUN_TIME_PY_VER_DETECT:7:4}; + if [[ ${VULNERABLE_VERSIONS[*]} =~ $PY_MAJOR_MINOR_VER ]]; then + rm -rf ${PIPX_HOME}/shared/lib/"python${PY_MAJOR_MINOR_VER}"/site-packages/setuptools-65.5.0.dist-info + fi + rm -rf /tmp/pip-tmp updaterc "export PIPX_HOME=\"${PIPX_HOME}\"" diff --git a/test/python/install_python310_setuptools_vulnerability.sh b/test/python/install_python310_setuptools_vulnerability.sh new file mode 100644 index 000000000..d6f68f1fb --- /dev/null +++ b/test/python/install_python310_setuptools_vulnerability.sh @@ -0,0 +1,89 @@ +#!/bin/bash +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +FAILED=() + +echoStderr() +{ + echo "$@" 1>&2 +} + +check-version-ge() { + LABEL=$1 + CURRENT_VERSION=$2 + REQUIRED_VERSION=$3 + shift + echo -e "\n🧪 Testing $LABEL: '$CURRENT_VERSION' is >= '$REQUIRED_VERSION'" + local GREATER_VERSION=$((echo ${CURRENT_VERSION}; echo ${REQUIRED_VERSION}) | sort -V | tail -1) + if [ "${CURRENT_VERSION}" == "${GREATER_VERSION}" ]; then + echo "✅ Passed!" + return 0 + else + echoStderr "❌ $LABEL check failed." + FAILED+=("$LABEL") + return 1 + fi +} + +checkPythonPackageVersion() +{ + PACKAGE=$1 + REQUIRED_VERSION=$2 + + current_version=$(python -c "import importlib.metadata; print(importlib.metadata.version('${PACKAGE}'))") + check-version-ge "${PACKAGE}-requirement" "${current_version}" "${REQUIRED_VERSION}" +} + +checkPythonPackageVersion "setuptools" "65.5.1" + +# Check that tools can execute - make sure something didn't get messed up in this scenario +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version +check "setuptools" pip list | grep setuptools + +# Check paths in settings +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +checkVulnerableDir() +{ + DIRECTORY=$1 + VERSION=$2 + + if [[ -d $DIRECTORY ]] ; then + echoStderr "❌ check for vulnerable setuptools version failed for python ${VERSION}." + return 1 + else + echo "✅ Passed! Either the container does not have vulnerable version or vulnerable version specific directory got removed." + return 0 + fi +} + +bash -c "echo -e -n '\n'"; +bash -c "echo -e 'Files/Folders related to setuptools :-'"; +bash -c "find / -name \"*setuptools*\";" + +# only for 3.10 +checkVulnerableDir "/usr/local/py-utils/shared/lib/python3.10/site-packages/setuptools-65.5.0.dist-info" "3.10" + +# Report result +reportResults diff --git a/test/python/install_python311_setuptools_vulnerability.sh b/test/python/install_python311_setuptools_vulnerability.sh new file mode 100644 index 000000000..b1e3983d2 --- /dev/null +++ b/test/python/install_python311_setuptools_vulnerability.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +FAILED=() + +echoStderr() +{ + echo "$@" 1>&2 +} + +check-version-ge() { + LABEL=$1 + CURRENT_VERSION=$2 + REQUIRED_VERSION=$3 + shift + echo -e "\n🧪 Testing $LABEL: '$CURRENT_VERSION' is >= '$REQUIRED_VERSION'" + local GREATER_VERSION=$((echo ${CURRENT_VERSION}; echo ${REQUIRED_VERSION}) | sort -V | tail -1) + if [ "${CURRENT_VERSION}" == "${GREATER_VERSION}" ]; then + echo "✅ Passed!" + return 0 + else + echoStderr "❌ $LABEL check failed." + FAILED+=("$LABEL") + return 1 + fi +} + +checkPythonPackageVersion() +{ + PACKAGE=$1 + REQUIRED_VERSION=$2 + + current_version=$(python -c "import importlib.metadata; print(importlib.metadata.version('${PACKAGE}'))") + check-version-ge "${PACKAGE}-requirement" "${current_version}" "${REQUIRED_VERSION}" +} + +checkPythonPackageVersion "setuptools" "65.5.1" + +# Check that tools can execute - make sure something didn't get messed up in this scenario +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version +check "pytest" pytest --version +check "setuptools" pip list | grep setuptools + +# Check paths in settings +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" +check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" + +checkVulnerableDir() +{ + DIRECTORY=$1 + VERSION=$2 + + if [[ -d $DIRECTORY ]] ; then + echoStderr "❌ check for vulnerable setuptools version failed for python ${VERSION}." + return 1 + else + echo "✅ Passed! Either the container does not have vulnerable version or vulnerable version specific directory got removed." + return 0 + fi +} + +bash -c "echo -e -n '\n'"; +bash -c "echo -e 'Files/Folders related to setuptools :-'"; +bash -c "find / -name \"*setuptools*\";" + +# only for 3.11 +checkVulnerableDir "/usr/local/py-utils/shared/lib/python3.11/site-packages/setuptools-65.5.0.dist-info" "3.11" + +# Report result +reportResults diff --git a/test/python/scenarios.json b/test/python/scenarios.json index a377dae1a..d89e8d573 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -1,4 +1,22 @@ { + "install_python310_setuptools_vulnerability": { + "image": "python:3.10", + "features": { + "python": { + "version": "none", + "installTools": true + } + } + }, + "install_python311_setuptools_vulnerability": { + "image": "python:3.11", + "features": { + "python": { + "version": "none", + "installTools": true + } + } + }, "install_additional_python": { "image": "ubuntu:focal", "features": { From 13a7f30d3971a2d37473b0b88281d81a1f29f797 Mon Sep 17 00:00:00 2001 From: Jeff Putsch Date: Sun, 4 Feb 2024 19:44:24 -0800 Subject: [PATCH 11/15] update check for managed python install. pass all tests. --- src/python/install.sh | 26 ++++++++++++------- ...n.sh => install_os_provided_python_deb.sh} | 0 ...tall_python310_setuptools_vulnerability.sh | 4 --- ...tall_python311_setuptools_vulnerability.sh | 4 --- test/python/scenarios.json | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) rename test/python/{install_os_provided_python.sh => install_os_provided_python_deb.sh} (100%) diff --git a/src/python/install.sh b/src/python/install.sh index a2ec5228f..8e08134ba 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -68,9 +68,9 @@ else exit 1 fi -# To find some devel packages, some rhel need to enable specific extra repos +# To find some devel packages, some rhel need to enable specific extra repos, but not on RedHat ubi images... INSTALL_CMD_ADDL_REPO="" -if [ ${ADJUSTED_ID} = "rhel" ]; then +if [ ${ADJUSTED_ID} = "rhel" ] && [ ${ID} != "rhel" ]; then if [ ${MAJOR_VERSION_ID} = "8" ]; then INSTALL_CMD_ADDL_REPOS="--enablerepo powertools" elif [ ${MAJOR_VERSION_ID} = "9" ]; then @@ -625,20 +625,16 @@ case ${ADJUSTED_ID} in ca-certificates \ findutils \ gcc \ - gdbm-devel \ gnupg2 \ libffi-devel \ libxml2-devel \ make \ ncurses-devel \ openssl-devel \ - readline-devel \ shadow-utils \ sqlite-devel \ tar \ - uuid-devel \ which \ - xmlsec1-devel \ xz-devel \ xz \ zlib-devel" @@ -646,11 +642,20 @@ case ${ADJUSTED_ID} in REQUIRED_PKGS="${REQUIRED_PKGS} \ curl" fi - # Mariner does not have tk-devel package available - if [ ${ID} != "mariner" ]; then + # Mariner does not have tk-devel package available, RedHat ubi8 and ubi9 do not have tk-devel + if [ ${ID} != "mariner" ] && [ ${ID} != "rhel" ]; then REQUIRED_PKGS="${REQUIRED_PKGS} \ tk-devel" fi + # Redhat ubi8 and ubi9 do not have some packages by default, only add them + # if we're not on RedHat ... + if [ ${ID} != "rhel" ]; then + REQUIRED_PKGS="${REQUIRED_PKGS} \ + gdbm-devel \ + readline-devel \ + uuid-devel \ + xmlsec1-devel" + fi ;; esac @@ -692,6 +697,9 @@ if [ "${PYTHON_VERSION}" != "none" ]; then find "${PYTHON_INSTALL_PATH}" -type d -print0 | xargs -0 -n 1 chmod g+s PYTHON_SRC="${INSTALL_PATH}/bin/python3" + if ! type pip >/dev/null 2>&1 && type pip3 >/dev/null 2>&1; then + ln -s /usr/bin/pip3 /usr/bin/pip + fi else PYTHON_SRC=$(which python) fi @@ -748,7 +756,7 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ -n "${PYTHON_SRC}" ]]; then # Temporary: Removes “setup tools” metadata directory due to https://github.com/advisories/GHSA-r9hx-vwmv-q579 VULNERABLE_VERSIONS=("3.10" "3.11") - RUN_TIME_PY_VER_DETECT=$(python --version 2>&1) + RUN_TIME_PY_VER_DETECT=$(${PYTHON_SRC} --version 2>&1) PY_MAJOR_MINOR_VER=${RUN_TIME_PY_VER_DETECT:7:4}; if [[ ${VULNERABLE_VERSIONS[*]} =~ $PY_MAJOR_MINOR_VER ]]; then rm -rf ${PIPX_HOME}/shared/lib/"python${PY_MAJOR_MINOR_VER}"/site-packages/setuptools-65.5.0.dist-info diff --git a/test/python/install_os_provided_python.sh b/test/python/install_os_provided_python_deb.sh similarity index 100% rename from test/python/install_os_provided_python.sh rename to test/python/install_os_provided_python_deb.sh diff --git a/test/python/install_python310_setuptools_vulnerability.sh b/test/python/install_python310_setuptools_vulnerability.sh index d6f68f1fb..0e065b1b3 100644 --- a/test/python/install_python310_setuptools_vulnerability.sh +++ b/test/python/install_python310_setuptools_vulnerability.sh @@ -78,10 +78,6 @@ checkVulnerableDir() fi } -bash -c "echo -e -n '\n'"; -bash -c "echo -e 'Files/Folders related to setuptools :-'"; -bash -c "find / -name \"*setuptools*\";" - # only for 3.10 checkVulnerableDir "/usr/local/py-utils/shared/lib/python3.10/site-packages/setuptools-65.5.0.dist-info" "3.10" diff --git a/test/python/install_python311_setuptools_vulnerability.sh b/test/python/install_python311_setuptools_vulnerability.sh index b1e3983d2..04a10948b 100644 --- a/test/python/install_python311_setuptools_vulnerability.sh +++ b/test/python/install_python311_setuptools_vulnerability.sh @@ -79,10 +79,6 @@ checkVulnerableDir() fi } -bash -c "echo -e -n '\n'"; -bash -c "echo -e 'Files/Folders related to setuptools :-'"; -bash -c "find / -name \"*setuptools*\";" - # only for 3.11 checkVulnerableDir "/usr/local/py-utils/shared/lib/python3.11/site-packages/setuptools-65.5.0.dist-info" "3.11" diff --git a/test/python/scenarios.json b/test/python/scenarios.json index d89e8d573..9e9d07e58 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -87,7 +87,7 @@ } } }, - "install_os_provided_python": { + "install_os_provided_python_deb": { "image": "mcr.microsoft.com/devcontainers/base:1-bullseye", "features": { "python": "os-provided" From dffe20031f217feee2d7c2ce18af882faa98a0c6 Mon Sep 17 00:00:00 2001 From: Jeff Putsch Date: Tue, 6 Feb 2024 13:20:37 -0800 Subject: [PATCH 12/15] add "packages" option from PR #768 --- .github/workflows/test-pr.yaml | 2 +- src/python/devcontainer-feature.json | 7 +- src/python/install.sh | 198 +++++++++--------- src/python/utils.sh | 27 +++ test/python/install_packages_mixed.sh | 23 ++ test/python/install_packages_non_root.sh | 23 ++ test/python/install_packages_root.sh | 19 ++ .../install_packages_versions_locked.sh | 23 ++ test/python/install_python_shared_lib.sh | 5 + test/python/install_python_shared_lib_deb.sh | 5 + .../install_python_shared_lib_rhel_family.sh | 5 + test/python/rocky-8-minimal.sh | 40 ---- test/python/rocky-8.sh | 40 ---- test/python/rocky-9-minimal.sh | 40 ---- test/python/rocky-9.sh | 40 ---- test/python/scenarios.json | 47 +++-- 16 files changed, 262 insertions(+), 282 deletions(-) create mode 100644 src/python/utils.sh create mode 100644 test/python/install_packages_mixed.sh create mode 100644 test/python/install_packages_non_root.sh create mode 100644 test/python/install_packages_root.sh create mode 100644 test/python/install_packages_versions_locked.sh delete mode 100755 test/python/rocky-8-minimal.sh delete mode 100755 test/python/rocky-8.sh delete mode 100755 test/python/rocky-9-minimal.sh delete mode 100755 test/python/rocky-9.sh diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml index d60733574..3aae841e2 100644 --- a/.github/workflows/test-pr.yaml +++ b/.github/workflows/test-pr.yaml @@ -67,7 +67,7 @@ jobs: test-scenarios: needs: [detect-changes] - runs-on: ubuntu-latest + runs-on: devcontainer-image-builder-ubuntu continue-on-error: true strategy: matrix: diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index 74ac1b676..14243d27b 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -25,13 +25,18 @@ "installTools": { "type": "boolean", "default": true, - "description": "Install common Python tools like pylint, override list using 'toolsToInstall'" + "description": "Install common Python tools like pylint, override list using 'toolsToInstall'. These are insatalled in virtual environments using pipx. The default list of tools is found in the `toolsToInstall` option." }, "toolsToInstall": { "type": "string", "default": "flake8,autopep8,black,yapf,mypy,pydocstyle,pycodestyle,bandit,pipenv,virtualenv,pytest,pylint", "description": "Comma-separated list of tools to install when installTools is true" }, + "packages": { + "type": "string", + "default": "", + "description": "Optional comma separated list of Python packages to install with pip." + }, "optimize": { "type": "boolean", "default": false, diff --git a/src/python/install.sh b/src/python/install.sh index 8e08134ba..9bee4c08f 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -27,9 +27,11 @@ CONFIGURE_JUPYTERLAB_ALLOW_ORIGIN="${CONFIGUREJUPYTERLABALLOWORIGIN:-""}" # alongside PYTHON_VERSION, but not set as default. ADDITIONAL_VERSIONS="${ADDITIONALVERSIONS:-""}" -GPG_INSTALL_PATH=/usr/local/gnupg22 -# GPG_CMD will be over-ridden if we need to install gpt2.22 ito GPG_INSTALL_PATH -GPG_CMD=gpg +# Comma-sparated list of packages to be installed +PYTHON_PACKAGES="${PACKAGES:-""}" + +# Import common utils +. ./utils.sh # Comma-separated list of additional tools to be installed via pipx. IFS="," read -r -a DEFAULT_UTILS <<< "${TOOLSTOINSTALL:-flake8,autopep8,black,yapf,mypy,pydocstyle,pycodestyle,bandit,pipenv,virtualenv,pytest}" @@ -133,7 +135,6 @@ updaterc() { fi } -# Import the specified key in a variable name passed in as # Import the specified key in a variable name passed in as receive_gpg_keys() { local keys=${!1} @@ -159,7 +160,53 @@ receive_gpg_keys() { until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; do echo "(*) Downloading GPG key..." - ( echo "${keys}" | xargs -n 1 ${GPG_CMD} -q ${keyring_args} --recv-keys) 2>&1 && gpg_ok="true" + ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys) 2>&1 && gpg_ok="true" + if [ "${gpg_ok}" != "true" ]; then + echo "(*) Failed getting key, retring in 10s..." + (( retry_count++ )) + sleep 10s + fi + done + set -e + if [ "${gpg_ok}" = "false" ]; then + echo "(!) Failed to get gpg key." + exit 1 + fi +} +# RHEL7/CentOS7 has an older gpg that does not have dirmngr +# Iterate through keyservers until we have all the keys downloaded +receive_gpg_keys_centos7() { + local keys=${!1} + local keyring_args="" + local gpg_cmd="gpg" + if [ ! -z "$2" ]; then + mkdir -p "$(dirname \"$2\")" + keyring_args="--no-default-keyring --keyring $2" + fi + if [ ! -z "${KEYSERVER_PROXY}" ]; then + keyring_args="${keyring_args} --keyserver-options http-proxy=${KEYSERVER_PROXY}" + fi + + # Use a temporary location for gpg keys to avoid polluting image + export GNUPGHOME="/tmp/tmp-gnupg" + mkdir -p ${GNUPGHOME} + chmod 700 ${GNUPGHOME} + # GPG key download sometimes fails for some reason and retrying fixes it. + local retry_count=0 + local gpg_ok="false" + num_keys=$(echo ${keys} | wc -w) + set +e + set -x + echo "(*) Downloading GPG keys..." + until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; do + for keyserver in ${GPG_KEY_SERVERS}; do + ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys --keyserver=${keyserver} ) 2>&1 + downloaded_keys=$(gpg --list-keys | grep ^pub | wc -l) + if [[ ${num_keys} = ${downloaded_keys} ]]; then + gpg_ok="true" + break + fi + done if [ "${gpg_ok}" != "true" ]; then echo "(*) Failed getting key, retring in 10s..." (( retry_count++ )) @@ -306,66 +353,19 @@ add_symlink() { fi } -install_openssl11() { +install_openssl3() { local _prefix=$1 - mkdir /tmp/openssl11 + mkdir /tmp/openssl3 ( - cd /tmp/openssl11 - curl -L -f -O https://www.openssl.org/source/openssl-1.1.1w.tar.gz - tar xzf openssl-1.1.1w.tar.gz - cd openssl-1.1.* - ./config --prefix=${_prefix} --openssldir=${_prefix} - make -j $(nproc) - make install_dev + cd /tmp/openssl3 + curl -L -f -O https://www.openssl.org/source/openssl-3.0.8.tar.gz + tar xzf openssl-3.0.8.tar.gz + cd openssl-* + ./config --prefix=${_prefix} --openssldir=${_prefix} + make -j $(nproc) + make install_dev ) - rm -rf /tmp/openssl11 -} - -install_gnupg22() { - local _prefix=$1 - yum-builddep -y gnupg2 - check_packages bzip2 - mkdir -p /tmp/gnupg22 && cd /tmp/gnupg22 - gpg --list-keys - gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 249B39D24F25E3B6 04376F3EE0856959 2071B08A33BD3F06 8A861B1C7EFD60D9 - - curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/libgpg-error/libgpg-error-1.31.tar.gz.sig && \ - curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/libgpg-error/libgpg-error-1.31.tar.gz && \ - curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.8.3.tar.gz && \ - curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/libgcrypt/libgcrypt-1.8.3.tar.gz.sig && \ - curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/libassuan/libassuan-2.5.1.tar.bz2 && \ - curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/libassuan/libassuan-2.5.1.tar.bz2.sig && \ - curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/libksba/libksba-1.3.5.tar.bz2 && \ - curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/libksba/libksba-1.3.5.tar.bz2.sig && \ - curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/npth/npth-1.5.tar.bz2 && \ - curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/npth/npth-1.5.tar.bz2.sig && \ - curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/gnupg/gnupg-2.2.9.tar.bz2 && \ - curl -sLfO -C - https://www.gnupg.org/ftp/gcrypt/gnupg/gnupg-2.2.9.tar.bz2.sig && \ - gpg --verify libgpg-error-1.31.tar.gz.sig && tar -xzf libgpg-error-1.31.tar.gz && \ - gpg --verify libgcrypt-1.8.3.tar.gz.sig && tar -xzf libgcrypt-1.8.3.tar.gz && \ - gpg --verify libassuan-2.5.1.tar.bz2.sig && tar -xjf libassuan-2.5.1.tar.bz2 && \ - gpg --verify libksba-1.3.5.tar.bz2.sig && tar -xjf libksba-1.3.5.tar.bz2 && \ - gpg --verify npth-1.5.tar.bz2.sig && tar -xjf npth-1.5.tar.bz2 && \ - gpg --verify gnupg-2.2.9.tar.bz2.sig && tar -xjf gnupg-2.2.9.tar.bz2 && \ - cd libgpg-error-1.31/ && ./configure --prefix=${_prefix} && make && make install && cd ../ && \ - cd libgcrypt-1.8.3 && ./configure --prefix=${_prefix} --with-libgpg-error-prefix=${_prefix} && make && make install && cd ../ && \ - cd libassuan-2.5.1 && ./configure --prefix=${_prefix} --with-libgpg-error-prefix=${_prefix} && make && make install && cd ../ && \ - cd libksba-1.3.5 && ./configure --prefix=${_prefix} && make && make install && cd ../ && \ - cd npth-1.5 && ./configure --prefix=${_prefix} && make && make install && cd ../ && \ - cd gnupg-2.2.9 && LDFLAGS="-Wl,-rpath=${_prefix}/lib" ./configure --prefix=${_prefix} \ - --with-agent-pgm=${_prefix}/bin/gpg-agent \ - --with-dirmngr-pgm=${_prefix}/bin/dirmngr \ - --with-libgpg-error-prefix=${_prefix} \ - --with-libgcrypt-prefix=${_prefix} \ - --with-libassuan-prefix=${_prefix} \ - --with-ksba-prefix=${_prefix} \ - --with-npth-prefix=${_prefix} \ - && make && make install && cd .. - # Without the line below, gpg2 might fail to create / import secret keys !!! - if [ -d ~/.gnugp ]; then rm -ri ~/.gnugp; fi - gpgconf --kill gpg-agent || true - # tidy up - rm -rf cd /tmp/gnupg22 + rm -rf /tmp/openssl3 } install_from_source() { @@ -385,14 +385,13 @@ install_from_source() { exit 1 fi - # Some platforms/os versions need openssl11 & gpg2.22 installed because they're available + # Some platforms/os versions need modern versions of openssl installed # via common package repositories, for now rhel-7 family, use case statement to # make it easy to expand case ${VERSION_CODENAME} in centos7|rhel7) - install_gnupg22 ${GPG_INSTALL_PATH} - GPG_CMD=${GPG_INSTALL_PATH}/bin/gpg - install_openssl11 ${INSTALL_PATH} + check_packages perl-IPC-Cmd + install_openssl3 ${INSTALL_PATH} ADDL_CONFIG_ARGS="--with-openssl=${INSTALL_PATH} --with-openssl-rpath=${INSTALL_PATH}/lib" ;; esac @@ -406,10 +405,14 @@ install_from_source() { curl -sSL -o "/tmp/python-src/${tgz_filename}" "${tgz_url}" # Verify signature - receive_gpg_keys PYTHON_SOURCE_GPG_KEYS + if [[ ${VERSION_CODENAME} = "centos7" ]] || [[ ${VERSION_CODENAME} = "rhel7" ]]; then + receive_gpg_keys_centos7 PYTHON_SOURCE_GPG_KEYS + else + receive_gpg_keys PYTHON_SOURCE_GPG_KEYS + fi echo "Downloading ${tgz_filename}.asc..." curl -sSL -o "/tmp/python-src/${tgz_filename}.asc" "${tgz_url}.asc" - ${GPG_CMD} --verify "${tgz_filename}.asc" + gpg --verify "${tgz_filename}.asc" # Update min protocol for testing only - https://bugs.python.org/issue41561 if [ -f /etc/pki/tls/openssl.cnf ]; then @@ -426,7 +429,6 @@ install_from_source() { if [ "${OPTIMIZE_BUILD_FROM_SOURCE}" = "true" ]; then config_args="${config_args} --enable-optimizations" fi - set -x if [ "${ENABLESHARED}" = "true" ]; then config_args=" ${config_args} --enable-shared" # need double-$: LDFLAGS ends up in Makefile $$ becomes $ when evaluated. @@ -473,25 +475,6 @@ install_using_oryx() { add_symlink } -sudo_if() { - COMMAND="$*" - if [ "$(id -u)" -eq 0 ] && [ "$USERNAME" != "root" ]; then - su - "$USERNAME" -c "$COMMAND" - else - $COMMAND - fi -} - -install_user_package() { - INSTALL_UNDER_ROOT="$1" - PACKAGE="$2" - - if [ "$INSTALL_UNDER_ROOT" = true ]; then - sudo_if "${PYTHON_SRC}" -m pip install --upgrade --no-cache-dir "$PACKAGE" - else - sudo_if "${PYTHON_SRC}" -m pip install --user --upgrade --no-cache-dir "$PACKAGE" - fi -} add_user_jupyter_config() { CONFIG_DIR="$1" @@ -732,18 +715,13 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ -n "${PYTHON_SRC}" ]]; then export PIP_CACHE_DIR=/tmp/pip-tmp/cache PIPX_DIR="" if ! type pipx > /dev/null 2>&1; then - - # python install is marked as externally managed, we need to pass - # pip a "--break-system-packages" option to avoid errors... This does not - # really breack system packages dueto the setting of PYTHONUSERBASE above, - # but does get us past checks for installing python packages into the - # system python install. if python_is_externally_managed ${PYTHON_SRC}; then - break_system_packages="--break-system-packages" + check_packages pipx + else + pip3 install ${break_system_packages} --disable-pip-version-check --no-cache-dir --user pipx 2>&1 + /tmp/pip-tmp/bin/pipx install --pip-args=--no-cache-dir pipx + PIPX_DIR="/tmp/pip-tmp/bin/" fi - pip3 install ${break_system_packages} --disable-pip-version-check --no-cache-dir --user pipx 2>&1 - /tmp/pip-tmp/bin/pipx install --pip-args=--no-cache-dir pipx - PIPX_DIR="/tmp/pip-tmp/bin/" fi for util in "${DEFAULT_UTILS[@]}"; do if ! type ${util} > /dev/null 2>&1; then @@ -769,6 +747,8 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ -n "${PYTHON_SRC}" ]]; then updaterc "if [[ \"\${PATH}\" != *\"\${PIPX_BIN_DIR}\"* ]]; then export PATH=\"\${PATH}:\${PIPX_BIN_DIR}\"; fi" fi +set -x + # Install JupyterLab if needed if [ "${INSTALL_JUPYTERLAB}" = "true" ]; then if [ -z "${PYTHON_SRC}" ]; then @@ -781,8 +761,8 @@ if [ "${INSTALL_JUPYTERLAB}" = "true" ]; then INSTALL_UNDER_ROOT=false fi - install_user_package $INSTALL_UNDER_ROOT jupyterlab - install_user_package $INSTALL_UNDER_ROOT jupyterlab-git + install_python_package $INSTALL_UNDER_ROOT $PYTHON_SRC jupyterlab + install_python_package $INSTALL_UNDER_ROOT $PYTHON_SRC jupyterlab-git # Configure JupyterLab if needed if [ -n "${CONFIGURE_JUPYTERLAB_ALLOW_ORIGIN}" ]; then @@ -799,6 +779,24 @@ if [ "${INSTALL_JUPYTERLAB}" = "true" ]; then fi fi +# Install pacakages if needed +if [ ! -z "${PYTHON_PACKAGES}" ]; then + if [ -z "${PYTHON_SRC}" ]; then + echo "(!) Could not install packages. Python not found." + exit 1 + fi + + INSTALL_UNDER_ROOT=true + if [ "$(id -u)" -eq 0 ] && [ "$USERNAME" != "root" ]; then + INSTALL_UNDER_ROOT=false + fi + + IFS="," read -a python_packages <<< "$PYTHON_PACKAGES" + for package in "${python_packages[@]}"; do + install_python_package $INSTALL_UNDER_ROOT $PYTHON_SRC $package + done +fi + # Clean up clean_up diff --git a/src/python/utils.sh b/src/python/utils.sh new file mode 100644 index 000000000..1bf20dc20 --- /dev/null +++ b/src/python/utils.sh @@ -0,0 +1,27 @@ +sudo_if() { + COMMAND="$*" + if [ "$(id -u)" -eq 0 ] && [ "$USERNAME" != "root" ]; then + su - "$USERNAME" -c "$COMMAND" + else + $COMMAND + fi +} + +install_python_package() { + INSTALL_UNDER_ROOT="$1" + PYTHON_PATH="$2" + PACKAGE="$3" + + sudo_if "$PYTHON_PATH" -m pip uninstall --yes "$PACKAGE" + + install_package="${PACKAGE}" + package_name=$(echo "${PACKAGE}" | sed 's/[<>=!~].*//') + + if [ "$INSTALL_UNDER_ROOT" = true ]; then + sudo_if "$PYTHON_PATH" -m pip install --upgrade --no-cache-dir "$install_package" + else + sudo_if "$PYTHON_PATH" -m pip install --upgrade --user --no-cache-dir "$install_package" + fi + + sudo_if "$PYTHON_PATH" -m pip --no-python-version-warning show "$package_name" +} \ No newline at end of file diff --git a/test/python/install_packages_mixed.sh b/test/python/install_packages_mixed.sh new file mode 100644 index 000000000..598d146b2 --- /dev/null +++ b/test/python/install_packages_mixed.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Always run these checks as the non-root user +user="$(whoami)" +check "user" grep vscode <<< "$user" + +# Check for an installation of JupyterLab +check "version" jupyter lab --version + +# Check location of JupyterLab installation +packages="$(python3 -m pip list)" +check "location" grep jupyter <<< "$packages" + +# Check for git extension +check "jupyterlab_git" grep jupyterlab_git <<< "$packages" + +# Report result +reportResults \ No newline at end of file diff --git a/test/python/install_packages_non_root.sh b/test/python/install_packages_non_root.sh new file mode 100644 index 000000000..598d146b2 --- /dev/null +++ b/test/python/install_packages_non_root.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Always run these checks as the non-root user +user="$(whoami)" +check "user" grep vscode <<< "$user" + +# Check for an installation of JupyterLab +check "version" jupyter lab --version + +# Check location of JupyterLab installation +packages="$(python3 -m pip list)" +check "location" grep jupyter <<< "$packages" + +# Check for git extension +check "jupyterlab_git" grep jupyterlab_git <<< "$packages" + +# Report result +reportResults \ No newline at end of file diff --git a/test/python/install_packages_root.sh b/test/python/install_packages_root.sh new file mode 100644 index 000000000..809ad4658 --- /dev/null +++ b/test/python/install_packages_root.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Check for an installation of JupyterLab +check "version" jupyter lab --version + +# Check location of JupyterLab installation +packages="$(python3 -m pip list)" +check "location" grep jupyter <<< "$packages" + +# Check for git extension +check "jupyterlab_git" grep jupyterlab_git <<< "$packages" + +# Report result +reportResults \ No newline at end of file diff --git a/test/python/install_packages_versions_locked.sh b/test/python/install_packages_versions_locked.sh new file mode 100644 index 000000000..598d146b2 --- /dev/null +++ b/test/python/install_packages_versions_locked.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Always run these checks as the non-root user +user="$(whoami)" +check "user" grep vscode <<< "$user" + +# Check for an installation of JupyterLab +check "version" jupyter lab --version + +# Check location of JupyterLab installation +packages="$(python3 -m pip list)" +check "location" grep jupyter <<< "$packages" + +# Check for git extension +check "jupyterlab_git" grep jupyterlab_git <<< "$packages" + +# Report result +reportResults \ No newline at end of file diff --git a/test/python/install_python_shared_lib.sh b/test/python/install_python_shared_lib.sh index 038826d5c..608a19851 100755 --- a/test/python/install_python_shared_lib.sh +++ b/test/python/install_python_shared_lib.sh @@ -36,5 +36,10 @@ check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bi check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" +# Check compiled with "--enable-shared" -- python truth is opposite of the shells +# therefore we negate the check to get a '0' exit code when the python +# has been compiled with --enable-shared +check "python shared library" /usr/local/python/bin/python -c 'import sys; import sysconfig; sys.exit(sysconfig.get_config_vars("Py_ENABLE_SHARED") != [1])' + # Report result reportResults diff --git a/test/python/install_python_shared_lib_deb.sh b/test/python/install_python_shared_lib_deb.sh index 038826d5c..608a19851 100755 --- a/test/python/install_python_shared_lib_deb.sh +++ b/test/python/install_python_shared_lib_deb.sh @@ -36,5 +36,10 @@ check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bi check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" +# Check compiled with "--enable-shared" -- python truth is opposite of the shells +# therefore we negate the check to get a '0' exit code when the python +# has been compiled with --enable-shared +check "python shared library" /usr/local/python/bin/python -c 'import sys; import sysconfig; sys.exit(sysconfig.get_config_vars("Py_ENABLE_SHARED") != [1])' + # Report result reportResults diff --git a/test/python/install_python_shared_lib_rhel_family.sh b/test/python/install_python_shared_lib_rhel_family.sh index 038826d5c..608a19851 100755 --- a/test/python/install_python_shared_lib_rhel_family.sh +++ b/test/python/install_python_shared_lib_rhel_family.sh @@ -36,5 +36,10 @@ check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bi check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" +# Check compiled with "--enable-shared" -- python truth is opposite of the shells +# therefore we negate the check to get a '0' exit code when the python +# has been compiled with --enable-shared +check "python shared library" /usr/local/python/bin/python -c 'import sys; import sysconfig; sys.exit(sysconfig.get_config_vars("Py_ENABLE_SHARED") != [1])' + # Report result reportResults diff --git a/test/python/rocky-8-minimal.sh b/test/python/rocky-8-minimal.sh deleted file mode 100755 index 038826d5c..000000000 --- a/test/python/rocky-8-minimal.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -set -e - -# Optional: Import test library -source dev-container-features-test-lib - -# Definition specific tests -check "version" python --version -check "pip is installed" pip --version -check "pip is installed" pip3 --version - -# Check that tools can execute -check "autopep8" autopep8 --version -check "black" black --version -check "yapf" yapf --version -check "bandit" bandit --version -check "flake8" flake8 --version -check "mypy" mypy --version -check "pycodestyle" pycodestyle --version -check "pydocstyle" pydocstyle --version -check "pylint" pylint --version -check "pytest" pytest --version - -# Check paths in settings -check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" -check "current symlink works" /usr/local/python/current/bin/python --version -check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" -check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" -check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" -check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" -check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" -check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" -check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" -check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" -check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" -check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" - -# Report result -reportResults diff --git a/test/python/rocky-8.sh b/test/python/rocky-8.sh deleted file mode 100755 index 038826d5c..000000000 --- a/test/python/rocky-8.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -set -e - -# Optional: Import test library -source dev-container-features-test-lib - -# Definition specific tests -check "version" python --version -check "pip is installed" pip --version -check "pip is installed" pip3 --version - -# Check that tools can execute -check "autopep8" autopep8 --version -check "black" black --version -check "yapf" yapf --version -check "bandit" bandit --version -check "flake8" flake8 --version -check "mypy" mypy --version -check "pycodestyle" pycodestyle --version -check "pydocstyle" pydocstyle --version -check "pylint" pylint --version -check "pytest" pytest --version - -# Check paths in settings -check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" -check "current symlink works" /usr/local/python/current/bin/python --version -check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" -check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" -check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" -check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" -check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" -check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" -check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" -check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" -check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" -check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" - -# Report result -reportResults diff --git a/test/python/rocky-9-minimal.sh b/test/python/rocky-9-minimal.sh deleted file mode 100755 index 038826d5c..000000000 --- a/test/python/rocky-9-minimal.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -set -e - -# Optional: Import test library -source dev-container-features-test-lib - -# Definition specific tests -check "version" python --version -check "pip is installed" pip --version -check "pip is installed" pip3 --version - -# Check that tools can execute -check "autopep8" autopep8 --version -check "black" black --version -check "yapf" yapf --version -check "bandit" bandit --version -check "flake8" flake8 --version -check "mypy" mypy --version -check "pycodestyle" pycodestyle --version -check "pydocstyle" pydocstyle --version -check "pylint" pylint --version -check "pytest" pytest --version - -# Check paths in settings -check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" -check "current symlink works" /usr/local/python/current/bin/python --version -check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" -check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" -check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" -check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" -check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" -check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" -check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" -check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" -check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" -check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" - -# Report result -reportResults diff --git a/test/python/rocky-9.sh b/test/python/rocky-9.sh deleted file mode 100755 index 038826d5c..000000000 --- a/test/python/rocky-9.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -set -e - -# Optional: Import test library -source dev-container-features-test-lib - -# Definition specific tests -check "version" python --version -check "pip is installed" pip --version -check "pip is installed" pip3 --version - -# Check that tools can execute -check "autopep8" autopep8 --version -check "black" black --version -check "yapf" yapf --version -check "bandit" bandit --version -check "flake8" flake8 --version -check "mypy" mypy --version -check "pycodestyle" pycodestyle --version -check "pydocstyle" pydocstyle --version -check "pylint" pylint --version -check "pytest" pytest --version - -# Check paths in settings -check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" -check "current symlink works" /usr/local/python/current/bin/python --version -check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" -check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" -check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" -check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" -check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" -check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" -check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" -check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" -check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" -check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" - -# Report result -reportResults diff --git a/test/python/scenarios.json b/test/python/scenarios.json index 9e9d07e58..6cd4fdde4 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -208,61 +208,68 @@ } } }, - "rocky-8": { - "image": "rockylinux:8", + "fedora": { + "image": "fedora", "features": { "python": { "version": "3.11" } } }, - "rocky-8-minimal": { - "image": "rockylinux:8-minimal", + "install_python_shared_lib_deb": { + "image": "ubuntu:focal", "features": { "python": { - "version": "3.11" + "version": "3.11", + "enableShared": true } } }, - "rocky-9": { - "image": "rockylinux:9", + "install_python_shared_lib_rhel_family": { + "image": "almalinux:8", "features": { "python": { - "version": "3.11" + "version": "3.11", + "enableShared": true } } }, - "rocky-9-minimal": { - "image": "rockylinux:9-minimal", + "install_packages_root": { + "image": "debian:bullseye-slim", "features": { "python": { - "version": "3.11" + "version": "3.11", + "packages": "jupyterlab,jupyterlab-git" } } }, - "fedora": { - "image": "fedora", + "install_packages_non_root": { + "image": "mcr.microsoft.com/devcontainers/base:focal", + "remoteUser": "vscode", "features": { "python": { - "version": "3.11" + "version": "3.11", + "packages": "jupyterlab,jupyterlab-git" } } }, - "install_python_shared_lib_deb": { - "image": "ubuntu:focal", + "install_packages_versions_locked": { + "image": "mcr.microsoft.com/devcontainers/base:focal", + "remoteUser": "vscode", "features": { "python": { "version": "3.11", - "enableShared": true + "packages": "cryptography==41.0.5,jupyterlab==4.0.8,jupyterlab-git==0.43.0" } } }, - "install_python_shared_lib_rhel_family": { - "image": "almalinux:8", + "install_packages_mixed": { + "image": "mcr.microsoft.com/devcontainers/base:focal", + "remoteUser": "vscode", "features": { "python": { "version": "3.11", - "enableShared": true + "packages": "cryptography==41.0.5,jupyterlab,jupyterlab-git==0.43.0" } } } From 1473900ce17605a46fa980346dcf3d3f887d7628 Mon Sep 17 00:00:00 2001 From: Jeff Putsch Date: Tue, 6 Feb 2024 14:02:55 -0800 Subject: [PATCH 13/15] remove "packages" option --- src/python/devcontainer-feature.json | 9 +---- src/python/install.sh | 6 --- src/python/utils.sh | 27 ------------- test/python/install_packages_mixed.sh | 23 ----------- test/python/install_packages_non_root.sh | 23 ----------- test/python/install_packages_root.sh | 19 --------- .../install_packages_versions_locked.sh | 23 ----------- test/python/scenarios.json | 39 ------------------- 8 files changed, 2 insertions(+), 167 deletions(-) delete mode 100644 src/python/utils.sh delete mode 100644 test/python/install_packages_mixed.sh delete mode 100644 test/python/install_packages_non_root.sh delete mode 100644 test/python/install_packages_root.sh delete mode 100644 test/python/install_packages_versions_locked.sh diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index 14243d27b..26f2e5a52 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -25,17 +25,12 @@ "installTools": { "type": "boolean", "default": true, - "description": "Install common Python tools like pylint, override list using 'toolsToInstall'. These are insatalled in virtual environments using pipx. The default list of tools is found in the `toolsToInstall` option." + "description": "Flag indicating whether or not to install the tools specified via the 'toolsToInstall' option. Default is 'true'." }, "toolsToInstall": { "type": "string", "default": "flake8,autopep8,black,yapf,mypy,pydocstyle,pycodestyle,bandit,pipenv,virtualenv,pytest,pylint", - "description": "Comma-separated list of tools to install when installTools is true" - }, - "packages": { - "type": "string", - "default": "", - "description": "Optional comma separated list of Python packages to install with pip." + "description": "Comma-separated list of tools to install when 'installTools' is true. Defaults to a set of common Python tools like pylint." }, "optimize": { "type": "boolean", diff --git a/src/python/install.sh b/src/python/install.sh index 9bee4c08f..0911823f9 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -27,12 +27,6 @@ CONFIGURE_JUPYTERLAB_ALLOW_ORIGIN="${CONFIGUREJUPYTERLABALLOWORIGIN:-""}" # alongside PYTHON_VERSION, but not set as default. ADDITIONAL_VERSIONS="${ADDITIONALVERSIONS:-""}" -# Comma-sparated list of packages to be installed -PYTHON_PACKAGES="${PACKAGES:-""}" - -# Import common utils -. ./utils.sh - # Comma-separated list of additional tools to be installed via pipx. IFS="," read -r -a DEFAULT_UTILS <<< "${TOOLSTOINSTALL:-flake8,autopep8,black,yapf,mypy,pydocstyle,pycodestyle,bandit,pipenv,virtualenv,pytest}" diff --git a/src/python/utils.sh b/src/python/utils.sh deleted file mode 100644 index 1bf20dc20..000000000 --- a/src/python/utils.sh +++ /dev/null @@ -1,27 +0,0 @@ -sudo_if() { - COMMAND="$*" - if [ "$(id -u)" -eq 0 ] && [ "$USERNAME" != "root" ]; then - su - "$USERNAME" -c "$COMMAND" - else - $COMMAND - fi -} - -install_python_package() { - INSTALL_UNDER_ROOT="$1" - PYTHON_PATH="$2" - PACKAGE="$3" - - sudo_if "$PYTHON_PATH" -m pip uninstall --yes "$PACKAGE" - - install_package="${PACKAGE}" - package_name=$(echo "${PACKAGE}" | sed 's/[<>=!~].*//') - - if [ "$INSTALL_UNDER_ROOT" = true ]; then - sudo_if "$PYTHON_PATH" -m pip install --upgrade --no-cache-dir "$install_package" - else - sudo_if "$PYTHON_PATH" -m pip install --upgrade --user --no-cache-dir "$install_package" - fi - - sudo_if "$PYTHON_PATH" -m pip --no-python-version-warning show "$package_name" -} \ No newline at end of file diff --git a/test/python/install_packages_mixed.sh b/test/python/install_packages_mixed.sh deleted file mode 100644 index 598d146b2..000000000 --- a/test/python/install_packages_mixed.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -e - -# Optional: Import test library -source dev-container-features-test-lib - -# Always run these checks as the non-root user -user="$(whoami)" -check "user" grep vscode <<< "$user" - -# Check for an installation of JupyterLab -check "version" jupyter lab --version - -# Check location of JupyterLab installation -packages="$(python3 -m pip list)" -check "location" grep jupyter <<< "$packages" - -# Check for git extension -check "jupyterlab_git" grep jupyterlab_git <<< "$packages" - -# Report result -reportResults \ No newline at end of file diff --git a/test/python/install_packages_non_root.sh b/test/python/install_packages_non_root.sh deleted file mode 100644 index 598d146b2..000000000 --- a/test/python/install_packages_non_root.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -e - -# Optional: Import test library -source dev-container-features-test-lib - -# Always run these checks as the non-root user -user="$(whoami)" -check "user" grep vscode <<< "$user" - -# Check for an installation of JupyterLab -check "version" jupyter lab --version - -# Check location of JupyterLab installation -packages="$(python3 -m pip list)" -check "location" grep jupyter <<< "$packages" - -# Check for git extension -check "jupyterlab_git" grep jupyterlab_git <<< "$packages" - -# Report result -reportResults \ No newline at end of file diff --git a/test/python/install_packages_root.sh b/test/python/install_packages_root.sh deleted file mode 100644 index 809ad4658..000000000 --- a/test/python/install_packages_root.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -set -e - -# Optional: Import test library -source dev-container-features-test-lib - -# Check for an installation of JupyterLab -check "version" jupyter lab --version - -# Check location of JupyterLab installation -packages="$(python3 -m pip list)" -check "location" grep jupyter <<< "$packages" - -# Check for git extension -check "jupyterlab_git" grep jupyterlab_git <<< "$packages" - -# Report result -reportResults \ No newline at end of file diff --git a/test/python/install_packages_versions_locked.sh b/test/python/install_packages_versions_locked.sh deleted file mode 100644 index 598d146b2..000000000 --- a/test/python/install_packages_versions_locked.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -e - -# Optional: Import test library -source dev-container-features-test-lib - -# Always run these checks as the non-root user -user="$(whoami)" -check "user" grep vscode <<< "$user" - -# Check for an installation of JupyterLab -check "version" jupyter lab --version - -# Check location of JupyterLab installation -packages="$(python3 -m pip list)" -check "location" grep jupyter <<< "$packages" - -# Check for git extension -check "jupyterlab_git" grep jupyterlab_git <<< "$packages" - -# Report result -reportResults \ No newline at end of file diff --git a/test/python/scenarios.json b/test/python/scenarios.json index 6cd4fdde4..d574c84e8 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -233,44 +233,5 @@ "enableShared": true } } - }, - "install_packages_root": { - "image": "debian:bullseye-slim", - "features": { - "python": { - "version": "3.11", - "packages": "jupyterlab,jupyterlab-git" - } - } - }, - "install_packages_non_root": { - "image": "mcr.microsoft.com/devcontainers/base:focal", - "remoteUser": "vscode", - "features": { - "python": { - "version": "3.11", - "packages": "jupyterlab,jupyterlab-git" - } - } - }, - "install_packages_versions_locked": { - "image": "mcr.microsoft.com/devcontainers/base:focal", - "remoteUser": "vscode", - "features": { - "python": { - "version": "3.11", - "packages": "cryptography==41.0.5,jupyterlab==4.0.8,jupyterlab-git==0.43.0" - } - } - }, - "install_packages_mixed": { - "image": "mcr.microsoft.com/devcontainers/base:focal", - "remoteUser": "vscode", - "features": { - "python": { - "version": "3.11", - "packages": "cryptography==41.0.5,jupyterlab,jupyterlab-git==0.43.0" - } - } } } \ No newline at end of file From baf1568d80bb6f1eb73dd7e1b774b2fa5f3b662d Mon Sep 17 00:00:00 2001 From: Jeff Putsch Date: Wed, 7 Feb 2024 06:15:28 -0800 Subject: [PATCH 14/15] Address PR feedback, passes all tests locally. --- src/python/install.sh | 41 ++++++++--------- test/python/install_python_shared_lib.sh | 45 ------------------- test/python/install_python_shared_lib_deb.sh | 2 +- .../install_python_shared_lib_rhel_family.sh | 2 +- 4 files changed, 23 insertions(+), 67 deletions(-) delete mode 100755 test/python/install_python_shared_lib.sh diff --git a/src/python/install.sh b/src/python/install.sh index 0911823f9..e8f9dd030 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -469,6 +469,25 @@ install_using_oryx() { add_symlink } +sudo_if() { + COMMAND="$*" + if [ "$(id -u)" -eq 0 ] && [ "$USERNAME" != "root" ]; then + su - "$USERNAME" -c "$COMMAND" + else + $COMMAND + fi +} + +install_user_package() { + INSTALL_UNDER_ROOT="$1" + PACKAGE="$2" + + if [ "$INSTALL_UNDER_ROOT" = true ]; then + sudo_if "${PYTHON_SRC}" -m pip install --upgrade --no-cache-dir "$PACKAGE" + else + sudo_if "${PYTHON_SRC}" -m pip install --user --upgrade --no-cache-dir "$PACKAGE" + fi +} add_user_jupyter_config() { CONFIG_DIR="$1" @@ -755,8 +774,8 @@ if [ "${INSTALL_JUPYTERLAB}" = "true" ]; then INSTALL_UNDER_ROOT=false fi - install_python_package $INSTALL_UNDER_ROOT $PYTHON_SRC jupyterlab - install_python_package $INSTALL_UNDER_ROOT $PYTHON_SRC jupyterlab-git + install_user_package $INSTALL_UNDER_ROOT jupyterlab + install_user_package $INSTALL_UNDER_ROOT jupyterlab-git # Configure JupyterLab if needed if [ -n "${CONFIGURE_JUPYTERLAB_ALLOW_ORIGIN}" ]; then @@ -773,24 +792,6 @@ if [ "${INSTALL_JUPYTERLAB}" = "true" ]; then fi fi -# Install pacakages if needed -if [ ! -z "${PYTHON_PACKAGES}" ]; then - if [ -z "${PYTHON_SRC}" ]; then - echo "(!) Could not install packages. Python not found." - exit 1 - fi - - INSTALL_UNDER_ROOT=true - if [ "$(id -u)" -eq 0 ] && [ "$USERNAME" != "root" ]; then - INSTALL_UNDER_ROOT=false - fi - - IFS="," read -a python_packages <<< "$PYTHON_PACKAGES" - for package in "${python_packages[@]}"; do - install_python_package $INSTALL_UNDER_ROOT $PYTHON_SRC $package - done -fi - # Clean up clean_up diff --git a/test/python/install_python_shared_lib.sh b/test/python/install_python_shared_lib.sh deleted file mode 100755 index 608a19851..000000000 --- a/test/python/install_python_shared_lib.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -set -e - -# Optional: Import test library -source dev-container-features-test-lib - -# Definition specific tests -check "version" python --version -check "pip is installed" pip --version -check "pip is installed" pip3 --version - -# Check that tools can execute -check "autopep8" autopep8 --version -check "black" black --version -check "yapf" yapf --version -check "bandit" bandit --version -check "flake8" flake8 --version -check "mypy" mypy --version -check "pycodestyle" pycodestyle --version -check "pydocstyle" pydocstyle --version -check "pylint" pylint --version -check "pytest" pytest --version - -# Check paths in settings -check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" -check "current symlink works" /usr/local/python/current/bin/python --version -check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" -check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" -check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" -check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" -check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" -check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" -check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" -check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" -check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" -check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest" - -# Check compiled with "--enable-shared" -- python truth is opposite of the shells -# therefore we negate the check to get a '0' exit code when the python -# has been compiled with --enable-shared -check "python shared library" /usr/local/python/bin/python -c 'import sys; import sysconfig; sys.exit(sysconfig.get_config_vars("Py_ENABLE_SHARED") != [1])' - -# Report result -reportResults diff --git a/test/python/install_python_shared_lib_deb.sh b/test/python/install_python_shared_lib_deb.sh index 608a19851..ee16c436a 100755 --- a/test/python/install_python_shared_lib_deb.sh +++ b/test/python/install_python_shared_lib_deb.sh @@ -39,7 +39,7 @@ check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest # Check compiled with "--enable-shared" -- python truth is opposite of the shells # therefore we negate the check to get a '0' exit code when the python # has been compiled with --enable-shared -check "python shared library" /usr/local/python/bin/python -c 'import sys; import sysconfig; sys.exit(sysconfig.get_config_vars("Py_ENABLE_SHARED") != [1])' +check "python enable shared" /usr/local/python/current/bin/python -c 'import sys; import sysconfig; sys.exit(sysconfig.get_config_vars("Py_ENABLE_SHARED") != [1])' # Report result reportResults diff --git a/test/python/install_python_shared_lib_rhel_family.sh b/test/python/install_python_shared_lib_rhel_family.sh index 608a19851..ee16c436a 100755 --- a/test/python/install_python_shared_lib_rhel_family.sh +++ b/test/python/install_python_shared_lib_rhel_family.sh @@ -39,7 +39,7 @@ check "which pytest" bash -c "which pytest | grep /usr/local/py-utils/bin/pytest # Check compiled with "--enable-shared" -- python truth is opposite of the shells # therefore we negate the check to get a '0' exit code when the python # has been compiled with --enable-shared -check "python shared library" /usr/local/python/bin/python -c 'import sys; import sysconfig; sys.exit(sysconfig.get_config_vars("Py_ENABLE_SHARED") != [1])' +check "python enable shared" /usr/local/python/current/bin/python -c 'import sys; import sysconfig; sys.exit(sysconfig.get_config_vars("Py_ENABLE_SHARED") != [1])' # Report result reportResults From 6c1335aa1c7b0788c13e2fe075d2abc49fcc5b8f Mon Sep 17 00:00:00 2001 From: Jeff Putsch Date: Wed, 7 Feb 2024 20:46:24 -0800 Subject: [PATCH 15/15] fix install error on centos --- src/python/install.sh | 51 +++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/src/python/install.sh b/src/python/install.sh index e8f9dd030..9094191f3 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -190,23 +190,22 @@ receive_gpg_keys_centos7() { local gpg_ok="false" num_keys=$(echo ${keys} | wc -w) set +e - set -x - echo "(*) Downloading GPG keys..." - until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; do - for keyserver in ${GPG_KEY_SERVERS}; do - ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys --keyserver=${keyserver} ) 2>&1 - downloaded_keys=$(gpg --list-keys | grep ^pub | wc -l) - if [[ ${num_keys} = ${downloaded_keys} ]]; then - gpg_ok="true" - break + echo "(*) Downloading GPG keys..." + until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; do + for keyserver in $(echo "${GPG_KEY_SERVERS}" | sed 's/keyserver //'); do + ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys --keyserver=${keyserver} ) 2>&1 + downloaded_keys=$(gpg --list-keys | grep ^pub | wc -l) + if [[ ${num_keys} = ${downloaded_keys} ]]; then + gpg_ok="true" + break + fi + done + if [ "${gpg_ok}" != "true" ]; then + echo "(*) Failed getting key, retring in 10s..." + (( retry_count++ )) + sleep 10s fi done - if [ "${gpg_ok}" != "true" ]; then - echo "(*) Failed getting key, retring in 10s..." - (( retry_count++ )) - sleep 10s - fi - done set -e if [ "${gpg_ok}" = "false" ]; then echo "(!) Failed to get gpg key." @@ -352,10 +351,16 @@ install_openssl3() { mkdir /tmp/openssl3 ( cd /tmp/openssl3 - curl -L -f -O https://www.openssl.org/source/openssl-3.0.8.tar.gz - tar xzf openssl-3.0.8.tar.gz - cd openssl-* - ./config --prefix=${_prefix} --openssldir=${_prefix} + openssl3_version="3.0" + # Find version using soft match + find_version_from_git_tags openssl3_version "https://github.com/openssl/openssl" "openssl-" + local tgz_filename="openssl-${openssl3_version}.tar.gz" + local tgz_url="https://github.com/openssl/openssl/releases/download/openssl-${openssl3_version}/${tgz_filename}" + echo "Downloading ${tgz_filename}..." + curl -sSL -o "/tmp/openssl3/${tgz_filename}" "${tgz_url}" + tar xzf ${tgz_filename} + cd openssl-${openssl3_version} + ./config --prefix=${_prefix} --openssldir=${_prefix} --libdir=lib make -j $(nproc) make install_dev ) @@ -363,7 +368,7 @@ install_openssl3() { } install_from_source() { - VERSION=$1 + VERSION=$1 echo "(*) Building Python ${VERSION} from source..." if ! type git > /dev/null 2>&1; then check_packages git @@ -435,7 +440,7 @@ install_from_source() { ./configure --prefix="${INSTALL_PATH}" --with-ensurepip=install ${config_args} make -j 8 make install - set +x + cd /tmp rm -rf /tmp/python-src ${GNUPGHOME} /tmp/vscdc-settings.env @@ -731,7 +736,7 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ -n "${PYTHON_SRC}" ]]; then if python_is_externally_managed ${PYTHON_SRC}; then check_packages pipx else - pip3 install ${break_system_packages} --disable-pip-version-check --no-cache-dir --user pipx 2>&1 + pip3 install --disable-pip-version-check --no-cache-dir --user pipx 2>&1 /tmp/pip-tmp/bin/pipx install --pip-args=--no-cache-dir pipx PIPX_DIR="/tmp/pip-tmp/bin/" fi @@ -760,8 +765,6 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ -n "${PYTHON_SRC}" ]]; then updaterc "if [[ \"\${PATH}\" != *\"\${PIPX_BIN_DIR}\"* ]]; then export PATH=\"\${PATH}:\${PIPX_BIN_DIR}\"; fi" fi -set -x - # Install JupyterLab if needed if [ "${INSTALL_JUPYTERLAB}" = "true" ]; then if [ -z "${PYTHON_SRC}" ]; then