[nrf noup] Added zap_append to west command

Added zap_append command to the west.

use `west zap-append -h` to see all options.

Now you can add new clusters to the matter ZCL datamodel.
Use a `west zap-append` and provide the new cluster
definitionsas XML files to add them to the zcl.json file.
You can generate the new zcl.json file by providing
`-o, --output` argument, or overwrite the existing one.

The zcl_generate is integrated with `west zap-gui` commands.
If you want to use it call the `west zap-gui` command with
the additional argument `--clusters`.
You can provide multiple clusters at once.
If you provide an additional `-j` / `--zcl-json` argument
alongside the `--clusters` argument to the `west zap-gui`
command, the new zcl.json will be created and contain
the new clusters.

Signed-off-by: Arkadiusz Balys <[email protected]>

scripts/west/west-commands.yml

@@ -10,3 +10,8 @@ west-commands:
       - name: zap-gui
         class: ZapGui
         help: Run Matter ZCL Advanced Platform (ZAP) GUI
+  - file: scripts/west/zap_append.py
+    commands:
+      - name: zap-append
+        class: ZapAppend
+        help: Append a new custom cluster to the ZCL database

scripts/west/zap_append.py

@@ -0,0 +1,98 @@
+# Copyright (c) 2025 Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
+
+import argparse
+from pathlib import Path
+import json
+
+from west.commands import WestCommand
+
+from zap_common import DEFAULT_MATTER_PATH, DEFAULT_ZCL_PATH, DEFAULT_DATA_MODEL_PATH
+
+
+def add_cluster_to_zcl(zcl_base: Path, cluster_xml_paths: list, output: Path):
+    """
+    Add the cluster to the ZCL file.
+    """
+
+    try:
+        with open(zcl_base, "r") as zcl_json_base:
+            zcl_json = json.load(zcl_json_base)
+    except IOError as e:
+        raise RuntimeError(f"No such ZCL file: {zcl_base}")
+
+    # If the output file is provided, we would like to generate a new ZCL file, so we must set
+    # the relative paths from the xml.file to the data model directories and manufacturers.xml file.
+    # It is because the base zcl.json file contains the relative paths to itself inside as the "xmlRoot",
+    # and if we create a new zcl file outside the data model directory it will not work properly.
+    if output:
+        roots_replaced = list()
+        replace = False
+
+        # Replace existing paths with the relative to output ones
+        for path in zcl_json.get("xmlRoot"):
+            path = zcl_base.parent.joinpath(Path(path))
+            if not path == "./" and not path == "." and not path.is_relative_to(output.parent):
+                roots_replaced.append(str(path.relative_to(output.parent, walk_up=True)))
+                replace = True
+        if replace:
+            zcl_json["xmlRoot"] = roots_replaced
+
+        # Add the relative path to manufacturers XML
+        manufacturers_xml = zcl_base.parent.joinpath(Path(zcl_json.get("manufacturersXml"))).resolve()
+        if not manufacturers_xml.parent.is_relative_to(output.parent):
+            zcl_json.update({"manufacturersXml": str(manufacturers_xml.relative_to(output.parent, walk_up=True))})
+
+    # Add the new clusters to the ZCL file
+    for cluster in cluster_xml_paths:
+        if not Path(cluster).exists():
+            raise RuntimeError(f"No such cluster file: {cluster}")
+
+        file = Path(cluster).name
+        if not output:
+            output = DEFAULT_ZCL_PATH
+
+        relative_path = Path(cluster).absolute().parent.relative_to(output.parent, walk_up=True)
+
+        # We need to add to things:
+        # 1. The absolute path to the directory where the new xml file exists to the xmlRoot array.
+        # 2. The new xml file name to the xmlFile array.
+        if not relative_path in zcl_json.get("xmlRoot"):
+            zcl_json.get("xmlRoot").append(str(relative_path))
+        if not file in zcl_json.get("xmlFile"):
+            zcl_json.get("xmlFile").append(file)
+            print(f"Successfully added {file}")
+
+    # If output file is not provided, we will edit the existing ZCL file
+    file_to_write = output if output else zcl_base
+
+    # Save the dumped JSON to the output file
+    with open(file_to_write, "w+") as zcl_output:
+        zcl_output.write(json.dumps(zcl_json, indent=4))
+
+
+class ZapAppend(WestCommand):
+    def __init__(self):
+        super().__init__(
+            'zap-append',
+            'Add a new custom cluster to the ZCL Matter data model file',
+            'A tool for adding a custom cluster to the ZCL Matter Data Model file according to the base ZCL file and custom clusters definitions.')
+
+    def do_add_parser(self, parser_adder) -> argparse.ArgumentParser:
+        parser = parser_adder.add_parser(self.name,
+                                         help=self.help,
+                                         formatter_class=argparse.RawDescriptionHelpFormatter,
+                                         description=self.description)
+        parser.add_argument("-b", "--base", type=Path, default=DEFAULT_ZCL_PATH,
+                            help=f"An absolute path to the base zcl.json file. If not provided the path will be set to {DEFAULT_ZCL_PATH}.")
+        parser.add_argument("-m", "--matter", type=Path, default=DEFAULT_MATTER_PATH,
+                            help=f"An absolute path to the Matter directory. If not set the path with be set to the {DEFAULT_MATTER_PATH}")
+        parser.add_argument("-o", "--output", type=Path, default=DEFAULT_ZCL_PATH,
+                            help=f"Output path to store the generated zcl.json file. If not provided the path will be set to the base zcl.json file ({DEFAULT_ZCL_PATH}).")
+        parser.add_argument("new_clusters", nargs='+',
+                            help="Paths to the XML files that contains the custom cluster definitions")
+        return parser
+
+    def do_run(self, args, unknown_args) -> None:
+        add_cluster_to_zcl(args.base.absolute(), args.new_clusters, args.output.absolute())

scripts/west/zap_common.py

@@ -3,14 +3,14 @@
 # SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
 
 import argparse
-import os
 import platform
 import re
 import shutil
 import stat
 import subprocess
 import tempfile
 import wget
+import json
 
 from collections import deque
 from pathlib import Path
@@ -20,6 +20,9 @@
 from west import log
 
 DEFAULT_MATTER_PATH = Path(__file__).parents[2]
+DEFAULT_DATA_MODEL_PATH = Path(f'{DEFAULT_MATTER_PATH}/src/app/zap-templates/zcl')
+DEFAULT_ZCL_PATH = Path(f'{DEFAULT_DATA_MODEL_PATH}/zcl.json')
+DEFAULT_APP_TEMPLATES_PATH = Path(f'{DEFAULT_MATTER_PATH}/src/app/zap-templates/app-templates.json')
 
 
 def find_zap(root: Path = Path.cwd(), max_depth: int = 2):
@@ -77,13 +80,43 @@ def existing_dir_path(arg: str) -> Path:
     raise argparse.ArgumentTypeError(f'invalid directory path: \'{arg}\'')
 
 
+def update_zcl_in_zap(zap_file: Path, zcl_json: Path):
+    """
+    In the .zap file, there is a relative path to the zcl.json file.
+    Use this function to update zcl.json path if needed.
+    """
+    with open(zap_file, 'r+') as file:
+        data = json.load(file)
+        packages = data.get("package")
+        for package in packages:
+            if package.get("type") == "zcl-properties":
+                package.update({"path": str(zcl_json.absolute().relative_to(zap_file.parent.absolute(), walk_up=True))})
+        file.seek(0)
+        json.dump(data, file, indent=2)
+        file.truncate()
+
+
+def check_zcl_in_zap(zap_file: Path, zcl_json: Path):
+    """
+    Check if the zcl.json path in the .zap file is correct.
+    """
+    with open(zap_file, 'r') as file:
+        data = json.load(file)
+        packages = data.get("package")
+        for package in packages:
+            if package.get("type") == "zcl-properties":
+                return zcl_json.parent.absolute().is_relative_to(zap_file.parent.absolute())
+    return False
+
+
 class ZapInstaller:
     INSTALL_DIR = Path('.zap-install')
     ZAP_URL_PATTERN = 'https://github.com/project-chip/zap/releases/download/v%04d.%02d.%02d-nightly/%s.zip'
 
     def __init__(self, matter_path: Path):
         self.matter_path = matter_path
         self.install_path = matter_path / ZapInstaller.INSTALL_DIR
+        self.cache_path = self.install_path / '.zap'
 
         def unzip_darwin(zip: Path, out: Path):
             subprocess.check_call(['unzip', zip, '-d', out])
@@ -211,6 +244,25 @@ def update_zap_if_needed(self) -> None:
             log.inf('Installing ZAP {}.{}.{}'.format(*recommended_version))
             self.install_zap(recommended_version)
 
+    def get_cache_path(self) -> Path:
+        """
+        Returns ZAP cache directory.
+        """
+        return self.cache_path
+
+    def set_cache_path(self, path: Path) -> None:
+        """
+        Sets the ZAP cache directory.
+        """
+        self.cache_path = path
+
+    def clear_cache(self) -> None:
+        """
+        Removes files in the ZAP cache directory.
+        """
+        if self.cache_path.exists():
+            shutil.rmtree(self.cache_path, ignore_errors=True)
+
     @staticmethod
     def set_exec_permission(path: Path) -> None:
         os.chmod(path, os.stat(path).st_mode | stat.S_IEXEC)

scripts/west/zap_gui.py

@@ -3,12 +3,14 @@
 # SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
 
 import argparse
+from pathlib import Path
 
 from textwrap import dedent
 
 from west.commands import WestCommand
 
-from zap_common import existing_file_path, existing_dir_path, find_zap, ZapInstaller, DEFAULT_MATTER_PATH
+from zap_common import existing_file_path, existing_dir_path, find_zap, ZapInstaller, DEFAULT_MATTER_PATH, DEFAULT_ZCL_PATH, DEFAULT_APP_TEMPLATES_PATH, update_zcl_in_zap, check_zcl_in_zap
+from zap_append import add_cluster_to_zcl
 
 
 class ZapGui(WestCommand):
@@ -31,10 +33,13 @@ def do_add_parser(self, parser_adder):
                                          description=self.description)
         parser.add_argument('-z', '--zap-file', type=existing_file_path,
                             help='Path to data model configuration file (*.zap)')
-        parser.add_argument('-j', '--zcl-json', type=existing_file_path,
-                            help='Path to data model definition file (zcl.json)')
+        parser.add_argument('-j', '--zcl-json', type=str,
+                            help='Path to data model definition file (zcl.json). If new clusters are added using --clusters, the new zcl.json file will be be created and used.')
         parser.add_argument('-m', '--matter-path', type=existing_dir_path,
                             default=DEFAULT_MATTER_PATH, help='Path to Matter SDK')
+        parser.add_argument('--clusters', nargs='+',
+                            help="Paths to the XML files that contain the external cluster definitions")
+        parser.add_argument('-c', '--clear', action='store_true', help='Clear the ZAP cache')
         return parser
 
     def do_run(self, args, unknown_args):
@@ -44,20 +49,43 @@ def do_run(self, args, unknown_args):
             zap_file_path = find_zap()
 
         if args.zcl_json:
-            zcl_json_path = args.zcl_json.absolute()
+            zcl_json_path = Path(args.zcl_json).absolute()
         else:
-            zcl_json_path = args.matter_path / 'src/app/zap-templates/zcl/zcl.json'
+            zcl_json_path = DEFAULT_ZCL_PATH
 
-        app_templates_path = args.matter_path / 'src/app/zap-templates/app-templates.json'
+        if args.clusters:
+            base_zcl = zcl_json_path if zcl_json_path.exists() else DEFAULT_ZCL_PATH
+            zcl_generator = add_cluster_to_zcl(base_zcl, args.clusters, zcl_json_path)
+
+        app_templates_path = DEFAULT_APP_TEMPLATES_PATH
+
+        if not zcl_json_path.exists():
+            print(f"ZCL file not found: {zcl_json_path}")
+            return
+
+        print(f"Using ZAP file: {zap_file_path}")
+        print(f"Using ZCL file: {zcl_json_path}")
+        print(f"Using app templates: {app_templates_path}")
 
         zap_installer = ZapInstaller(args.matter_path)
         zap_installer.update_zap_if_needed()
-        zap_cache_path = zap_installer.get_install_path() / ".zap"
+        zap_cache_path = zap_installer.get_cache_path()
+
+        # The zcl.json path in the .zap file must be the same as the one provided by the user
+        # If not, update the .zap file with the new relative path to the zcl.json file.
+        # After that we must clear the ZAP cache.
+        if not check_zcl_in_zap(zap_file_path, zcl_json_path):
+            update_zcl_in_zap(zap_file_path, zcl_json_path)
+            args.clear = True
+
+        # clear zap cache before the usage
+        if args.clear:
+            print(f"Clearing ZAP cache: {zap_cache_path}")
+            zap_installer.clear_cache()
 
         cmd = [zap_installer.get_zap_path()]
         cmd += [zap_file_path] if zap_file_path else []
         cmd += ["--zcl", zcl_json_path]
         cmd += ["--gen", app_templates_path]
         cmd += ["--stateDirectory", zap_cache_path]
-
         self.check_call([str(x) for x in cmd])