Tools: Improve the Python package system

Introduce features into the Python package management system & manage
package versions outside of ESP-IDF repo.

.gitlab-ci.yml

@@ -72,6 +72,13 @@ variables:
   CI_AUTO_TEST_SCRIPT_REPO_BRANCH: "ci/v4.1"
   PYTEST_EMBEDDED_TAG: "v0.4.5"
 
+  # cache python dependencies
+  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+
+cache:
+  paths:
+    - .cache/pip
+
 .setup_tools_unless_target_test: &setup_tools_unless_target_test |
   if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
   export IDF_MIRROR_PREFIX_MAP=
@@ -95,6 +102,7 @@ before_script:
   - source tools/ci/configure_ci_environment.sh
   - *setup_tools_unless_target_test
   - fetch_submodules
+  - $IDF_PATH/tools/idf_tools.py install-python-env
 
 # used for check scripts which we want to run unconditionally
 .before_script_no_sync_submodule:
@@ -103,6 +111,7 @@ before_script:
     - source tools/ci/utils.sh
     - source tools/ci/setup_python.sh
     - source tools/ci/configure_ci_environment.sh
+    - $IDF_PATH/tools/idf_tools.py install-python-env
 
 .before_script_minimal:
   before_script:
@@ -133,6 +142,7 @@ before_script:
     - source tools/ci/configure_ci_environment.sh
     - *setup_tools_unless_target_test
     - fetch_submodules
+    - $IDF_PATH/tools/idf_tools.py install-python-env
     - cd /tmp
     - retry_failed git clone --depth 1 --branch $PYTEST_EMBEDDED_TAG https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/idf/pytest-embedded.git
     - cd pytest-embedded && bash foreach.sh install

.gitlab/CODEOWNERS

@@ -199,7 +199,7 @@
 
 /tools/unit-test-app/         @esp-idf-codeowners/system @esp-idf-codeowners/tools
 
-requirements.txt              @esp-idf-codeowners/tools
+requirements.*.txt              @esp-idf-codeowners/tools
 
 # sort-order-reset
 

.gitlab/ci/host-test.yml

@@ -215,8 +215,7 @@ test_idf_tools:
     - cd ${IDF_PATH}/tools/test_idf_tools
     - ./test_idf_tools.py
     # Test for create virtualenv. It must be invoked from Python, not from virtualenv.
-    - cd ${IDF_PATH}/tools
-    - python3 ./idf_tools.py install-python-env
+    - python3 ./test_idf_tools_python_env.py
 
 .test_efuse_table_on_host_template:
   extends: .host_test_template

.gitlab/ci/rules.yml

@@ -124,6 +124,7 @@
   - "tools/tools_schema.json"
   - "tools/idf_tools.py"
   - "tools/test_idf_tools/**/*"
+  - "tools/install_util.py"
 
   - "tools/mkdfu.py"
   - "tools/test_mkdfu/**/*"

docs/en/api-guides/jtag-debugging/using-debugger.rst

@@ -228,7 +228,7 @@ It is also possible to execute the described debugging tools conveniently from `
 
 4.  ``idf.py gdbgui``
 
-    Starts `gdbgui <https://www.gdbgui.com>`_ debugger frontend enabling out-of-the-box debugging in a browser window.
+    Starts `gdbgui <https://www.gdbgui.com>`_ debugger frontend enabling out-of-the-box debugging in a browser window. Please run the install script with the "--enable-gdbgui" argument in order to make this option supported, e.g. ``install.sh --enable-gdbgui``.
 
 
     It is possible to combine these debugging actions on a single command line allowing convenient setup of blocking and non-blocking actions in one step. ``idf.py`` implements a simple logic to move the background actions (such as openocd) to the beginning and the interactive ones (such as gdb, monitor) to the end of the action list.

docs/en/api-guides/tools/idf-tools.rst

@@ -101,6 +101,10 @@ Any mirror server can be used provided the URL matches the ``github.com`` downlo
 
 * ``check``: For each tool, checks whether the tool is available in the system path and in ``IDF_TOOLS_PATH``.
 
+* ``install-python-env``: Create Python virtual environment and install the required Python packages.
+
+* ``check-python-dependencies``: Checks if all required Python packages are installed.
+
 .. _idf-tools-install:
 
 Install scripts

export.bat

@@ -50,7 +50,7 @@ DOSKEY otatool.py=python.exe "%IDF_PATH%\components\app_update\otatool.py" $*
 DOSKEY parttool.py=python.exe "%IDF_PATH%\components\partition_table\parttool.py" $*
 
 echo Checking if Python packages are up to date...
-python.exe "%IDF_PATH%\tools\check_python_dependencies.py"
+python.exe "%IDF_PATH%\tools\idf_tools.py" check-python-dependencies
 if %errorlevel% neq 0 goto :__end
 
 echo.

export.fish

@@ -19,7 +19,7 @@ function __main
     eval "$idf_exports"
 
     echo "Checking if Python packages are up to date..."
-    python "$IDF_PATH"/tools/check_python_dependencies.py || return 1
+    python "$IDF_PATH"/tools/idf_tools.py check-python-dependencies || return 1
 
     # Allow calling some IDF python tools without specifying the full path
     # "$IDF_PATH"/tools is already added by 'idf_tools.py export'

export.ps1

@@ -69,7 +69,7 @@ if ($dif_Path -ne $null) {
 
 Write-Output "Checking if Python packages are up to date..."
 
-Start-Process -Wait -NoNewWindow -FilePath "python" -Args "`"$IDF_PATH/tools/check_python_dependencies.py`""
+Start-Process -Wait -NoNewWindow -FilePath "python" -Args "`"$IDF_PATH/tools/idf_tools.py`" check-python-dependencies"
 if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # if error
 
 Write-Output "

export.sh

@@ -96,7 +96,7 @@ __main() {
 
     __verbose "Using Python interpreter in $(which python)"
     __verbose "Checking if Python packages are up to date..."
-    python "${IDF_PATH}/tools/check_python_dependencies.py" || return 1
+    python "${IDF_PATH}/tools/idf_tools.py" check-python-dependencies || return 1
 
 
     # Allow calling some IDF python tools without specifying the full path

install.bat

@@ -17,15 +17,16 @@ if not "%MISSING_REQUIREMENTS%" == "" goto :error_missing_requirements
 set IDF_PATH=%~dp0
 set IDF_PATH=%IDF_PATH:~0,-1%
 
-set TARGETS="all"
-if NOT "%1"=="" set TARGETS=%*
+for /f "delims=" %%i in ('python.exe "%IDF_PATH%\tools\install_util.py" extract targets "%*"') do set TARGETS=%%i
 
 echo Installing ESP-IDF tools
 python.exe "%IDF_PATH%\tools\idf_tools.py" install --targets=%TARGETS%
 if %errorlevel% neq 0 goto :end
 
+for /f "delims=" %%i in ('python.exe "%IDF_PATH%\tools\install_util.py" extract features "%*"') do set FEATURES=%%i
+
 echo Setting up Python environment
-python.exe "%IDF_PATH%\tools\idf_tools.py" install-python-env
+python.exe "%IDF_PATH%\tools\idf_tools.py" install-python-env --features=%FEATURES%
 if %errorlevel% neq 0 goto :end
 
 echo All done! You can now run:

install.fish

@@ -7,17 +7,16 @@ set -x IDF_PATH $basedir
 echo "Detecting the Python interpreter"
 source "$IDF_PATH"/tools/detect_python.fish
 
-if not set -q argv[1]
-    set TARGETS "all"
-else
-    set TARGETS $argv[1]
-end
+set TARGETS ("$ESP_PYTHON" "$IDF_PATH"/tools/install_util.py extract targets $argv) || exit 1
+
 echo "Installing ESP-IDF tools"
 "$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py install --targets=$TARGETS
 or exit 1
 
+set FEATURES ("$ESP_PYTHON" "$IDF_PATH"/tools/install_util.py extract features $argv) || exit 1
+
 echo "Installing Python environment and packages"
-"$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py install-python-env
+"$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py install-python-env --features=$FEATURES
 
 echo "All done! You can now run:"
 echo ""

install.ps1

@@ -1,18 +1,16 @@
 #!/usr/bin/env pwsh
 $IDF_PATH = $PSScriptRoot
 
-if($args.count -eq 0){
-    $TARGETS = "all"
-}else
-{
-    $TARGETS = $args[0] -join ','
-}
+$TARGETS = (python "$IDF_PATH/tools/install_util.py" extract targets "$args")
+
 Write-Output "Installing ESP-IDF tools"
 Start-Process -Wait -NoNewWindow -FilePath "python" -Args "$IDF_PATH/tools/idf_tools.py install --targets=${TARGETS}"
 if ($LASTEXITCODE -ne 0)  { exit $LASTEXITCODE } # if error
 
+$FEATURES = (python "$IDF_PATH/tools/install_util.py" extract features "$args")
+
 Write-Output "Setting up Python environment"
-Start-Process -Wait  -NoNewWindow -FilePath "python" -Args "$IDF_PATH/tools/idf_tools.py install-python-env"
+Start-Process -Wait  -NoNewWindow -FilePath "python" -Args "$IDF_PATH/tools/idf_tools.py install-python-env --features=${FEATURES}"
 if ($LASTEXITCODE -ne 0)  { exit $LASTEXITCODE} # if error
 
 

install.sh

@@ -10,16 +10,15 @@ export IDF_PATH
 echo "Detecting the Python interpreter"
 . "${IDF_PATH}/tools/detect_python.sh"
 
-if [ "$#" -eq 0 ]; then
-  TARGETS="all"
-else
-  TARGETS=$1
-fi
+TARGETS=`"${ESP_PYTHON}" "${IDF_PATH}/tools/install_util.py" extract targets "$@"`
+
 echo "Installing ESP-IDF tools"
 "${ESP_PYTHON}" "${IDF_PATH}/tools/idf_tools.py" install --targets=${TARGETS}
 
+FEATURES=`"${ESP_PYTHON}" "${IDF_PATH}/tools/install_util.py" extract features "$@"`
+
 echo "Installing Python environment and packages"
-"${ESP_PYTHON}" "${IDF_PATH}/tools/idf_tools.py" install-python-env
+"${ESP_PYTHON}" "${IDF_PATH}/tools/idf_tools.py" install-python-env --features=${FEATURES}
 
 echo "All done! You can now run:"
 echo ""

requirements.core.txt

@@ -0,0 +1,23 @@
+# Python package requirements for ESP-IDF. These are the so called core features which are installed in all systems.
+
+setuptools
+click
+pyserial
+future
+cryptography
+pyparsing
+pyelftools
+idf-component-manager
+
+# esptool dependencies (see components/esptool_py/esptool/setup.py)
+reedsolo
+bitstring
+ecdsa
+
+# espcoredump dependencies
+construct
+pygdbmi
+
+# kconfig and menuconfig dependencies
+kconfiglib
+windows-curses; sys_platform == 'win32'

requirements.gdbgui.txt

@@ -0,0 +1,4 @@
+# Python package requirements for gdbgui support ESP-IDF.
+# This feature can be enabled by running "install.{sh,bat,ps1,fish} --enable-gdbgui"
+
+gdbgui