tools: Remove tools that are not used by active ESP-IDF version.

Or remove unused archives from tools - older version, or unused tool archive

docs/en/api-guides/tools/idf-tools.rst

@@ -108,6 +108,11 @@ Any mirror server can be used provided the URL matches the ``github.com`` downlo
 
 * ``check-python-dependencies``: Checks if all required Python packages are installed. Packages from ``${IDF_PATH}/tools/requirements/requirements.*.txt`` files selected by the feature list of ``idf-env.json`` are checked with the package versions specified in the ``espidf.constraints.*.txt`` file. The constraint file will be downloaded from https://dl.espressif.com if this step hasn't been done already in the last day.
 
+* ``uninstall``: Print and remove tools, that are currently not used by active ESP-IDF version.
+
+  - ``--dry-run`` Print installed unused tools.
+  - ``--remove-archives`` Additionally remove all older versions of previously downloaded installation packages.
+
 .. _idf-tools-install:
 
 Install scripts

export.bat

@@ -56,6 +56,11 @@ echo Checking if Python packages are up to date...
 python.exe "%IDF_PATH%\tools\idf_tools.py" check-python-dependencies
 if %errorlevel% neq 0 goto :__end
 
+python.exe "%IDF_PATH%\tools\idf_tools.py" uninstall --dry-run > UNINSTALL_OUTPUT
+SET /p UNINSTALL=<UNINSTALL_OUTPUT
+DEL UNINSTALL_OUTPUT
+if NOT "%UNINSTALL%"=="" call :__uninstall_message
+
 echo.
 echo Done! You can now compile ESP-IDF projects.
 echo Go to the project directory and run:
@@ -83,6 +88,13 @@ goto :__end
     echo For more details please visit our website: https://docs.espressif.com/projects/esp-idf/en/latest/esp32/get-started/windows-setup.html
     goto :__end
 
+:__uninstall_message
+    echo.
+    echo Detected installed tools that are not currently used by active ESP-IDF version.
+    echo %UNINSTALL%
+    echo For free up even more space, remove installation packages of those tools. Use option 'python.exe %IDF_PATH%\tools\idf_tools.py uninstall --remove-archives'.
+    echo.
+
 :__end
 :: Clean up
 if not "%IDF_TOOLS_EXPORTS_FILE%"=="" (
@@ -96,3 +108,4 @@ set IDF_TOOLS_JSON_PATH=
 set OLD_PATH=
 set PATH_ADDITIONS=
 set MISSING_REQUIREMENTS=
+set UNINSTALL=

export.fish

@@ -47,6 +47,15 @@ function __main
         echo "All paths are already set."
     end
 
+    set uninstall ("$ESP_PYTHON" "$IDF_PATH"/tools/idf_tools.py uninstall --dry-run) || return 1
+    if test -n "$uninstall"
+        echo ""
+        echo "Detected installed tools that are not currently used by active ESP-IDF version."
+        echo "$uninstall"
+        echo "For free up even more space, remove installation packages of those tools. Use option '$ESP_PYTHON $IDF_PATH/tools/idf_tools.py uninstall --remove-archives'."
+        echo ""
+    end
+
     # Clean up
     set -e added_path_variables
     set -e cmd
@@ -57,6 +66,7 @@ function __main
     set -e IDF_ADD_PATHS_EXTRAS
     set -e idf_exports
     set -e ESP_PYTHON
+    set -e uninstall
 
     # Not unsetting IDF_PYTHON_ENV_PATH, it can be used by IDF build system
     # to check whether we are using a private Python environment

export.ps1

@@ -75,6 +75,16 @@ Write-Output "Checking if Python packages are up to date..."
 Start-Process -Wait -NoNewWindow -FilePath "python" -Args "`"$IDF_PATH/tools/idf_tools.py`" check-python-dependencies"
 if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # if error
 
+$uninstall = python $IDF_PATH/tools/idf_tools.py uninstall --dry-run
+
+if (![string]::IsNullOrEmpty($uninstall)){
+    Write-Output ""
+    Write-Output "Detected installed tools that are not currently used by active ESP-IDF version."
+    Write-Output "$uninstall"
+    Write-Output "For free up even more space, remove installation packages of those tools. Use option 'python.exe $IDF_PATH\tools\idf_tools.py uninstall --remove-archives'."
+    Write-Output ""
+}
+
 Write-Output "
 Done! You can now compile ESP-IDF projects.
 Go to the project directory and run:

export.sh

@@ -132,6 +132,15 @@ __main() {
         __verbose "  ${PATH}"
     fi
 
+    uninstall=$("$ESP_PYTHON" "${IDF_PATH}/tools/idf_tools.py" uninstall --dry-run) || return 1
+    if [ -n "$uninstall" ]
+    then
+        __verbose ""
+        __verbose "Detected installed tools that are not currently used by active ESP-IDF version."
+        __verbose "${uninstall}"
+        __verbose "For free up even more space, remove installation packages of those tools. Use option '${ESP_PYTHON} ${IDF_PATH}/tools/idf_tools.py uninstall --remove-archives'."
+        __verbose ""
+    fi
 
     __verbose "Done! You can now compile ESP-IDF projects."
     __verbose "Go to the project directory and run:"
@@ -151,6 +160,7 @@ __cleanup() {
     unset SOURCE_ZSH
     unset SOURCE_BASH
     unset WARNING_MSG
+    unset uninstall
 
     unset __realpath
     unset __main

tools/idf_tools.py

@@ -977,6 +977,16 @@ def dump_tools_json(tools_info):  # type: ignore
     return json.dumps(file_json, indent=2, separators=(',', ': '), sort_keys=True)
 
 
+def get_python_exe_and_subdir() -> Tuple[str, str]:
+    if sys.platform == 'win32':
+        subdir = 'Scripts'
+        python_exe = 'python.exe'
+    else:
+        subdir = 'bin'
+        python_exe = 'python'
+    return python_exe, subdir
+
+
 def get_python_env_path():  # type: () -> Tuple[str, str, str, str]
     python_ver_major_minor = '{}.{}'.format(sys.version_info.major, sys.version_info.minor)
 
@@ -1018,13 +1028,7 @@ def get_python_env_path():  # type: () -> Tuple[str, str, str, str]
     idf_python_env_path = os.path.join(global_idf_tools_path, 'python_env',  # type: ignore
                                        'idf{}_py{}_env'.format(idf_version, python_ver_major_minor))
 
-    if sys.platform == 'win32':
-        subdir = 'Scripts'
-        python_exe = 'python.exe'
-    else:
-        subdir = 'bin'
-        python_exe = 'python'
-
+    python_exe, subdir = get_python_exe_and_subdir()
     idf_python_export_path = os.path.join(idf_python_env_path, subdir)
     virtualenv_python = os.path.join(idf_python_export_path, python_exe)
 
@@ -1374,39 +1378,53 @@ def apply_github_assets_option(tool_download_obj):  # type: ignore
         tool_download_obj.url = new_url
 
 
+def get_tools_spec_and_platform_info(selected_platform, targets, tools_spec,
+                                     quiet=False):  # type: (str, list[str], list[str], bool) -> Tuple[list[str], Dict[str, IDFTool]]
+    if selected_platform not in PLATFORM_FROM_NAME:
+        fatal(f'unknown platform: {selected_platform}')
+        raise SystemExit(1)
+    selected_platform = PLATFORM_FROM_NAME[selected_platform]
+
+    # If this function is not called from action_download, but is used just for detecting active tools, info about downloading is unwanted.
+    global global_quiet
+    try:
+        old_global_quiet = global_quiet
+        global_quiet = quiet
+        tools_info = load_tools_info()
+        tools_info_for_platform = OrderedDict()
+        for name, tool_obj in tools_info.items():
+            tool_for_platform = tool_obj.copy_for_platform(selected_platform)
+            tools_info_for_platform[name] = tool_for_platform
+
+        if not tools_spec or 'required' in tools_spec:
+            # Downloading tools for all ESP_targets required by the operating system.
+            tools_spec = [k for k, v in tools_info_for_platform.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS]
+            # Filtering tools user defined list of ESP_targets
+            if 'all' not in targets:
+                def is_tool_selected(tool):  # type: (IDFTool) -> bool
+                    supported_targets = tool.get_supported_targets()
+                    return (any(item in targets for item in supported_targets) or supported_targets == ['all'])
+                tools_spec = [k for k in tools_spec if is_tool_selected(tools_info[k])]
+            info('Downloading tools for {}: {}'.format(selected_platform, ', '.join(tools_spec)))
+
+        # Downloading tools for all ESP_targets (MacOS, Windows, Linux)
+        elif 'all' in tools_spec:
+            tools_spec = [k for k, v in tools_info_for_platform.items() if v.get_install_type() != IDFTool.INSTALL_NEVER]
+            info('Downloading tools for {}: {}'.format(selected_platform, ', '.join(tools_spec)))
+    finally:
+        global_quiet = old_global_quiet
+
+    return tools_spec, tools_info_for_platform
+
+
 def action_download(args):  # type: ignore
-    tools_info = load_tools_info()
     tools_spec = args.tools
     targets = []  # type: list[str]
     # Installing only single tools, no targets are specified.
     if 'required' in tools_spec:
         targets = add_and_save_targets(args.targets)
 
-    if args.platform not in PLATFORM_FROM_NAME:
-        fatal('unknown platform: {}' % args.platform)
-        raise SystemExit(1)
-    platform = PLATFORM_FROM_NAME[args.platform]
-
-    tools_info_for_platform = OrderedDict()
-    for name, tool_obj in tools_info.items():
-        tool_for_platform = tool_obj.copy_for_platform(platform)
-        tools_info_for_platform[name] = tool_for_platform
-
-    if not tools_spec or 'required' in tools_spec:
-        # Downloading tools for all ESP_targets required by the operating system.
-        tools_spec = [k for k, v in tools_info_for_platform.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS]
-        # Filtering tools user defined list of ESP_targets
-        if 'all' not in targets:
-            def is_tool_selected(tool):  # type: (IDFTool) -> bool
-                supported_targets = tool.get_supported_targets()
-                return (any(item in targets for item in supported_targets) or supported_targets == ['all'])
-            tools_spec = [k for k in tools_spec if is_tool_selected(tools_info[k])]
-        info('Downloading tools for {}: {}'.format(platform, ', '.join(tools_spec)))
-
-    # Downloading tools for all ESP_targets (MacOS, Windows, Linux)
-    elif 'all' in tools_spec:
-        tools_spec = [k for k, v in tools_info_for_platform.items() if v.get_install_type() != IDFTool.INSTALL_NEVER]
-        info('Downloading tools for {}: {}'.format(platform, ', '.join(tools_spec)))
+    tools_spec, tools_info_for_platform = get_tools_spec_and_platform_info(args.platform, targets, args.tools)
 
     for tool_spec in tools_spec:
         if '@' not in tool_spec:
@@ -1747,6 +1765,75 @@ def action_rewrite(args):  # type: ignore
     info('Wrote output to {}'.format(args.output))
 
 
+def action_uninstall(args):  # type: (Any) -> None
+    """ Print or remove installed tools, that are currently not used by active ESP-IDF version.
+    Additionally remove all older versions of previously downloaded archives.
+    """
+
+    def is_tool_selected(tool):  # type: (IDFTool) -> bool
+        supported_targets = tool.get_supported_targets()
+        return (supported_targets == ['all'] or any(item in targets for item in supported_targets))
+
+    tools_info = load_tools_info()
+    targets, _ = get_requested_targets_and_features()
+    tools_path = os.path.join(global_idf_tools_path or '', 'tools')
+    dist_path = os.path.join(global_idf_tools_path or '', 'dist')
+    used_tools = [k for k, v in tools_info.items() if (v.get_install_type() == IDFTool.INSTALL_ALWAYS and is_tool_selected(tools_info[k]))]
+    installed_tools = os.listdir(tools_path) if os.path.isdir(tools_path) else []
+    unused_tools = [tool for tool in installed_tools if tool not in used_tools]
+    # Keeping tools added by windows installer
+    KEEP_WIN_TOOLS = ['idf-git', 'idf-python']
+    for tool in KEEP_WIN_TOOLS:
+        if tool in unused_tools:
+            unused_tools.remove(tool)
+
+    # Print unused tools.
+    if args.dry_run:
+        if unused_tools:
+            print('For removing {} use command \'{} {} {}\''.format(', '.join(unused_tools), get_python_exe_and_subdir()[0],
+                  os.path.join(global_idf_path or '', 'tools', 'idf_tools.py'), 'uninstall'))
+        return
+
+    # Remove installed tools that are not used by current ESP-IDF version.
+    for tool in unused_tools:
+        try:
+            shutil.rmtree(os.path.join(tools_path, tool))
+            info(os.path.join(tools_path, tool) + ' was removed.')
+        except OSError as error:
+            warn(f'{error.filename} can not be removed because {error.strerror}.')
+
+    # Remove old archives versions and archives that are not used by the current ESP-IDF version.
+    if args.remove_archives:
+        targets, _ = get_requested_targets_and_features()
+        tools_spec, tools_info_for_platform = get_tools_spec_and_platform_info(CURRENT_PLATFORM, targets, ['required'], quiet=True)
+        used_archives = []
+
+        # Detect used active archives
+        for tool_spec in tools_spec:
+            if '@' not in tool_spec:
+                tool_name = tool_spec
+                tool_version = None
+            else:
+                tool_name, tool_version = tool_spec.split('@', 1)
+            tool_obj = tools_info_for_platform[tool_name]
+            if tool_version is None:
+                tool_version = tool_obj.get_recommended_version()
+            # mypy-checks
+            if tool_version is not None:
+                archive_version = tool_obj.versions[tool_version].get_download_for_platform(CURRENT_PLATFORM)
+            if archive_version is not None:
+                archive_version_url = archive_version.url
+
+            archive = os.path.basename(archive_version_url)
+            used_archives.append(archive)
+
+        downloaded_archives = os.listdir(dist_path)
+        for archive in downloaded_archives:
+            if archive not in used_archives:
+                os.remove(os.path.join(dist_path, archive))
+                info(os.path.join(dist_path, archive) + ' was removed.')
+
+
 def action_validate(args):  # type: ignore
     try:
         import jsonschema
@@ -1882,6 +1969,10 @@ def main(argv):  # type: (list[str]) -> None
     download.add_argument('--targets', default='all', help='A comma separated list of desired chip targets for installing.' +
                           ' It defaults to installing all supported targets.')
 
+    uninstall = subparsers.add_parser('uninstall', help='Remove installed tools, that are not used by current version of ESP-IDF.')
+    uninstall.add_argument('--dry-run', help='Print unused tools.', action='store_true')
+    uninstall.add_argument('--remove-archives', help='Remove old archive versions and archives from unused tools.', action='store_true')
+
     if IDF_MAINTAINER:
         for subparser in [download, install]:
             subparser.add_argument('--mirror-prefix-map', nargs='*',

tools/test_idf_tools/test_idf_tools.py

@@ -92,6 +92,7 @@ def setUpClass(cls):
 
         print('Using IDF_TOOLS_PATH={}'.format(cls.temp_tools_dir))
         os.environ['IDF_TOOLS_PATH'] = cls.temp_tools_dir
+        cls.idf_env_json = os.path.join(cls.temp_tools_dir, 'idf-env.json')
 
     @classmethod
     def tearDownClass(cls):
@@ -104,8 +105,8 @@ def tearDown(self):
         if os.path.isdir(os.path.join(self.temp_tools_dir, 'tools')):
             shutil.rmtree(os.path.join(self.temp_tools_dir, 'tools'))
 
-        if os.path.isfile(os.path.join(self.temp_tools_dir, 'idf-env.json')):
-            os.remove(os.path.join(self.temp_tools_dir, 'idf-env.json'))
+        if os.path.isfile(self.idf_env_json):
+            os.remove(self.idf_env_json)
 
     def assert_tool_installed(self, output, tool, tool_version, tool_archive_name=None):
         if tool_archive_name is None:
@@ -314,6 +315,23 @@ def test_tools_for_esp32s3(self):
         self.assertNotIn('%s/tools/xtensa-esp32s2-elf/%s/xtensa-esp32s2-elf/bin' %
                          (self.temp_tools_dir, XTENSA_ESP32S2_ELF_VERSION), output)
 
+    def test_uninstall_option(self):
+        self.run_idf_tools_with_action(['install', '--targets=esp32,esp32c3'])
+        output = self.run_idf_tools_with_action(['uninstall', '--dry-run'])
+        self.assertEqual(output, '')
+
+        with open(self.idf_env_json, 'r') as idf_env_file:
+            idf_env_json = json.load(idf_env_file)
+        idf_env_json['idfInstalled'][idf_env_json['idfSelectedId']]['targets'].remove('esp32')
+        with open(self.idf_env_json, 'w') as w:
+            json.dump(idf_env_json, w)
+
+        output = self.run_idf_tools_with_action(['uninstall'])
+        self.assertIn(XTENSA_ESP32_ELF, output)
+        self.assertIn(ESP32ULP, output)
+        output = self.run_idf_tools_with_action(['uninstall', '--dry-run'])
+        self.assertEqual(output, '')
+
 
 class TestMaintainer(unittest.TestCase):