Transform project to new file structure

.dockerignore

@@ -2,8 +2,13 @@
 *
 
 # Except this...
-!ifex
-!scripts
+!input_filters/
+!models/
+!output_filters/
+!packaging/
+!scripts/
+!tests/
+!transformers/
 !setup.py
-!tests
 !requirements.txt
+!tox.ini

.github/workflows/buildcheck.yml

@@ -63,8 +63,8 @@ jobs:
 
       - name: Run JSON-Schema generator
         run: |
-          python ifex/schema/ifex_to_json_schema.py >temp-schema
-          python ifex/schema/pretty_print_json.py temp-schema >ifex-core-idl-schema.json
+          python output_filters/schema/ifex_to_json_schema.py >temp-schema
+          python output_filters/schema/pretty_print_json.py temp-schema >ifex-core-idl-schema.json
 
       - name: Clone uservices repository
         run: |

.github/workflows/containers.yml

@@ -9,23 +9,23 @@ jobs:
 
       - name: Build containers
         run: |
-          cd docker
+          cd packaging/docker
           make build_alpine
           make build_ubuntu
 
       - name: Test Interactive Commands
         run: |
-          cd docker
+          cd packaging/docker
           variant=alpine make run_interactivity_test
           variant=ubuntu make run_interactivity_test_pyenv
 
       - name: Run unit tests in Ubuntu container
         run: |
-          cd docker
+          cd packaging/docker
           make run_ubuntu_test
 
       - name: Run unit tests in Alpine container
         run: |
-          cd docker
+          cd packaging/docker
           make run_alpine_test
 

.github/workflows/create-new-release.yml

@@ -3,14 +3,17 @@
 # Preconditions:
 # 1) The tag that is determined from the commit message must not already exist
 # 2) The workflow does not work if pushing an existing tag.  
-# 3) The precondition ensures this workflow is NOT run if no RELEASE-TAG field.
+# 3) The precondition ensures this workflow is NOT run if a RELEASE-TAG is
+#    not found in the commit message.
 #
 # Action steps:
-# 1) Commit and add the RELEASE-TAG in message
-# 2) Tag locally with the tag
-#   DO NOT PUSH THE TAG
+# 1) Set package version number in setup.py and add RELEASE-TAG: line to the commit message
+# 2) Tag locally with the tag name
+#   DO NOT PUSH THE TAG! *
 # 3) Push commit (possibly via a pull request)
-# 4) When pushed or merge to master, the release process will create the remote tag and the release
+# 4) When pushed or merged to master, the release process will create the remote tag and the release
+#    (Note: Since we use and expect fast-forward merges, "merging" here means fast forwarding
+#     and therefore the local tag should be on the same commit as the one created remotely.)
 
 name: Create release tag and publish JSON Schema
 
@@ -48,8 +51,8 @@ jobs:
 
       - name: Run JSON-Schema generator
         run: |
-          python ifex/schema/ifex_to_json_schema.py >temp-schema
-          python ifex/schema/pretty_print_json.py temp-schema >ifex-core-idl-schema.json
+          python output_filters/schema/ifex_to_json_schema.py >temp-schema
+          python output_filters/schema/pretty_print_json.py temp-schema >ifex-core-idl-schema.json
           sed -i 's/TAG-PLACEHOLDER/${{ steps.vars.outputs.TAG }}/' ifex-core-idl-schema.json
 
       - name: Create a new release

.github/workflows/generate_docs.yml

@@ -23,7 +23,7 @@ jobs:
 
       - name: Generate syntax document from source
         run: |
-          python ifex/model/ifex_ast_doc.py >docs/generated-syntax.md
+          python models/ifex/ifex_ast_doc.py >docs/generated-syntax.md
 
       - name: Join docs into one specification
         run: |

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "input_filters/franca/pyfranca"]
+	path = input_filters/franca/pyfranca
+	url = https://github.com/gunnarx/pyfranca

README.md

@@ -36,7 +36,7 @@ or [tox.ini](./tox.ini))
 
 ### Container use
 
-As an alternative to installation instructions below, all the installations can also be hidden in a container.  Refer to the [README in the docker/ directory](./docker/README.md) for running the tools using containers instead.
+As an alternative to installation instructions below, all the installations can also be hidden in a container.  Refer to the [README in the packaging/docker/ directory](./packaging/docker/README.md) for running the tools using containers instead.
 
 ## Installing and use python version(s) with `pyenv`
 

docs/README2.md

@@ -6,14 +6,14 @@ The documentation, [published](https://covesa.github.io/ifex) on GitHub pages, i
 
 - The [GitHub actions workflow](/.github/workflows/generate_docs.yml) encodes the build steps.
 - [markup-markdown](https://github.com/hailiang-wang/markup-markdown) is used to combine several markdown files into one.  Some markdown files are static and stored in [[doc-parts]], whereas some are generated from source files.  The naming convention for the definition files are `*.m.md`
-- [ifex_ast_doc.py](../ifex/model/ifex_ast_doc.py) generates the main syntax documentation directly from the datatypes used in the internal model AST.  The language definition for the YAML based IDL represents the internal object model.  In the end, both the program behavior _and the documentation of it_ is defined from the python source definition in [ifex_ast.py](../ifex/model/ifex_ast.py)
+- [ifex_ast_doc.py](../models/ifex/ifex_ast_doc.py) generates the main syntax documentation directly from the datatypes used in the internal model AST.  The language definition for the YAML based IDL represents the internal object model.  In the end, both the program behavior _and the documentation of it_ is defined from the python source definition in [ifex_ast.py](../models/ifex/ifex_ast.py)
 - The script [create-toc.py](create-toc.py) is used to generate a Table of Contents for the document.
 
 ## Source files:
 
 - [def-specification.stage1.m.md](./def-specification.stage1.m.md).  Lists the parts required to produce the main specification.
 - [def-specification.stage2.m.md](./def-specification.stage2.m.md).  To combine the specification with the table-of-contents.
-- [ifex_ast.py](../ifex/model/ifex_ast.py) is the python source that is interpreted to produce `generated-syntax.md` which is later included as the Syntax chapter in the specification.
+- [ifex_ast.py](../models/ifex/ifex_ast.py) is the python source that is interpreted to produce `generated-syntax.md` which is later included as the Syntax chapter in the specification.
 - [def-developers-manual.m.md](./def-developers-manual.m.md) lists the parts required to produce the developer documentation. (Work in progress)
 
 ## Build sequence

docs/generate-types-doc.py

@@ -3,7 +3,7 @@
 # SPDX-FileCopyrightText: Copyright (c) 2023 Novaspring AB
 # SPDX-License-Identifier: MPL-2.0
 
-from ifex.model.ifex_ast import FundamentalTypes
+from models.ifex.ifex_ast import FundamentalTypes
 
 print("|Name|Description|Min value|Max value|")
 print("|----|-----------|---------|---------|")

docs/static-developer-generators.md

@@ -39,7 +39,7 @@ A simple generator (with only one template) can be done like this:
 
 Unless you need to add more logic, generating one input file with one (or several)
 templates is basically already available if ifex_generator.py is called as a
-main program.  You specify the directory name (relative to `<project-dir>/ifex/templates`)
+main program.  You specify the directory name (relative to `<project-dir>/output_filters/templates`)
 where the template(s) is stored.  Note that `setup.py` "installs" an executable entrypoint
 `ifexgen` to call the program:
 
@@ -53,7 +53,7 @@ An advanced generator (with several templates) can be done like this:
 * Import the ifex_generator.py and ifex_parser.py, and TemplateDir modules.
 * Get the needed Service description file(s) (YAML), for example from command line argument
 * For each file, get the Abstract Syntax Tree representation by calling `ifex_parser.get_ast_from_yaml_file(service_desc_file)`
-* Write templates according to (some, not all) node types.  You can call `gen(node)` or `gen(node, <Template>)` from within a template - see details below.  Templates must be under a specific sub-directory of `<project-dir>/ifex/templates` and must be named with the naming convention such that the node type is mentioned first (see template chapter).
+* Write templates according to (some, not all) node types.  You can call `gen(node)` or `gen(node, <Template>)` from within a template - see details below.  Templates must be under a specific sub-directory of `<project-dir>/output_filters/templates` and must be named with the naming convention such that the node type is mentioned first (see template chapter).
 * Once the template directory is known, ee-initialize the TemplateDir object if needed.  This will populate the default templates data structure such that gen() knows which template to use.  It automatically uses the naming of the templates to know which template is for which node type.
 * Implement the `gen()` function, normally by delegating directly into `ifex_generator.gen()`, but you can put your own logic here if needed.
 * Set the jinja environment, passing in the gen() and any other symbols that must be callable from within Jinja templates.
@@ -143,7 +143,7 @@ gen(node, 'My-alternative-method-template.tpl')
 ### Naming convention
 The only strict requirement is that all templates to be used by automatic type-to-template mapping in the gen() function, must be within the same subdirectory and must be named according to the naming conventioned mentioned before.
 
-Templates can be stored in the sub-directories of the ifex/templates/ directory.
+Templates can be stored in the sub-directories of the output_filters/templates/ directory.
 
 The intended target format (output format) is normally clear from the naming of the subdirectory.  E.g. "protobuf", or "D-Bus" or "ARXML" or other output format.
 

ifex/scripts/diff_ifex.py → helpers/ifex/diff_ifex.py

@@ -10,7 +10,7 @@
 import tempfile
 import yaml
 from collections import OrderedDict
-from ifex.model.stable_sort_ifex import ifex_stable_order, represent_ordereddict
+from models.ifex.stable_sort_ifex import ifex_stable_order, represent_ordereddict
 
 # The program compares two IFEX (YAML) files after normalizing ("sorting",
 # basically) the order of elements so that the comparison becomes more relevant.

other/protobuf/test_convert_uservices.sh → helpers/test_convert_uservices.sh

@@ -1,8 +1,14 @@
 #!/bin/bash
 
+# A small test script to use https://github.com/COVESA/uservices
+# as a quite comprehensive input test suite, (it is defined in protobuf/gRPC IDL)
+# and convert as much as possible using the protobuf/grpc->IFEX converter
+
 # Normalize current directory
 cd "$(dirname "$0")"
 
+P2I=../input_filters/protobuf_to_ifex.py
+
 # Clone if not existing
 if [ ! -d uservices ] ; then
    (set -x ; git clone https://github.com/COVESA/uservices)
@@ -17,7 +23,7 @@ echo "Running protobuf_to_ifex on all uservices proto files"
 find uservices/ -name '*.proto' | while read f ; do 
    echo "=== $f ===" 
    ifexname="$(echo "$(basename "$f")" | sed 's/.proto/.ifex/g')"
-   python protobuf_to_ifex.py "$f" >".output/$ifexname"
+   python $P2I "$f" >".output/$ifexname"
    if [ $? -eq 0 ] ; then
       echo "$f" >>.ok
    else

input_filters/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 MBition GmbH.
+# SPDX-License-Identifier: MPL-2.0

other/franca/franca_to_ifex.py → input_filters/franca/franca_to_ifex.py

@@ -12,14 +12,14 @@
     if p not in sys.path:
         sys.path.append(os.path.join(mydir,p))
 
-import ifex.model.ifex_ast as ifex
-import other.franca.pyfranca.pyfranca as pyfranca
-import other.franca.rule_translator as m2m
+import models.ifex.ifex_ast as ifex
+import input_filters.franca.pyfranca.pyfranca as pyfranca
+import transformers.rule_translator as m2m
 import pyfranca.ast as franca
 import re
 
-from ifex.model.ifex_ast_construction import add_constructors_to_ifex_ast_model, ifex_ast_as_yaml
-from other.franca.rule_translator import Preparation, Constant, Unsupported
+from models.ifex.ifex_ast_construction import add_constructors_to_ifex_ast_model, ifex_ast_as_yaml
+from transformers.rule_translator import Preparation, Constant, Unsupported
 
 def translate_type_name(francaitem):
     return translate_type(francaitem)

other/franca/print_franca_tree.py → input_filters/franca/print_franca_tree.py

@@ -15,7 +15,7 @@
         sys.path.append(os.path.join(mydir,p))
 
 import oyaml
-import other.franca.pyfranca.pyfranca as pyfranca
+import input_filters.franca.pyfranca.pyfranca as pyfranca
 from collections import OrderedDict
 
 def is_simple_type(t) -> bool:

other/protobuf/protobuf_to_ifex.py → input_filters/protobuf/protobuf_to_ifex.py

@@ -3,10 +3,10 @@
 
 # This file is part of the IFEX project
 
-from ifex.model.ifex_ast_construction import add_constructors_to_ifex_ast_model, ifex_ast_as_yaml
-from other.protobuf.protobuf_lark import create_proto_ast
-import ifex.model.ifex_ast as ifex
-import other.protobuf.protobuf_ast as pb
+from models.ifex.ifex_ast_construction import add_constructors_to_ifex_ast_model, ifex_ast_as_yaml
+from models.protobuf.protobuf_lark import create_proto_ast
+import models.ifex.ifex_ast as ifex
+import models.protobuf.protobuf_ast as pb
 import os
 import sys
 

ifex/generators/DBus/dbus_types.py → models/DBus/dbus_types.py

@@ -4,7 +4,7 @@
 # Type resolver for the D-Bus XML format generator
 # ----------------------------------------------------------------------------
 
-from ifex.model.ifex_ast import (
+from models.ifex.ifex_ast import (
     AST,
     Namespace,
     Interface,
@@ -13,7 +13,7 @@
     Struct,
     Typedef,
 )
-from ifex.model import ifex_parser
+from models.ifex import ifex_parser
 import sys
 
 # D-Bus basic types, as defined in [D-Bus Specification](https://dbus.freedesktop.org/doc/dbus-specification.html#basic-types)

models/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 MBition GmbH.
+# SPDX-License-Identifier: MPL-2.0

ifex/model/ifex_ast_construction.py → models/ifex/ifex_ast_construction.py

@@ -26,8 +26,8 @@
 # With __init__ it is possible to create an object tree in a straight forward
 # and expected way, including some type checks:
 #
-#    from ifex.model import ifex_ast_construction 
-#    from ifex.model.ifex_ast import Namespace, Interface, ...
+#    from models.ifex import ifex_ast_construction 
+#    from models.ifex.ifex_ast import Namespace, Interface, ...
 #
 #    # Initialize support:
 #    ifex_ast_construction.add_constructors_to_ifex_ast_model()
@@ -45,8 +45,8 @@
 from collections import OrderedDict
 from dataclasses import is_dataclass, fields
 from datetime import date, datetime
-from ifex.model import ifex_ast
-from ifex.model.type_checking_constructor_mixin import add_constructor
+from models.ifex import ifex_ast
+from models.ifex.type_checking_constructor_mixin import add_constructor
 
 # Use the oyaml library because it supports OrderedDict. We can output keys in
 # the order they are defined in the AST classes (e.g. "name" comes first!)
@@ -113,7 +113,7 @@ def ifex_ast_as_yaml(node):
 # ----------------- TEST CODE BELOW --------------------
 
 if __name__ == '__main__':
-    from ifex.model.ifex_ast import AST, Namespace, Interface, Method, Argument
+    from models.ifex.ifex_ast import AST, Namespace, Interface, Method, Argument
 
     # How to create a AST representation in code:
     root = AST('test')

ifex/model/ifex_ast_doc.py → models/ifex/ifex_ast_doc.py

@@ -12,8 +12,8 @@
 """
 
 from dataclasses import fields
-from ifex.model.ifex_ast import Namespace
-from ifex.model.ifex_ast_introspect import walk_type_tree, field_is_list, is_optional, type_name, field_actual_type, field_inner_type
+from models.ifex.ifex_ast import Namespace
+from models.ifex.ifex_ast_introspect import walk_type_tree, field_is_list, is_optional, type_name, field_actual_type, field_inner_type
 import re,itertools
 
 #

ifex/model/ifex_ast_introspect.py → models/ifex/ifex_ast_introspect.py

@@ -15,7 +15,7 @@
 """
 
 import re
-import ifex.model.ifex_ast as ifex_ast
+import models.ifex.ifex_ast as ifex_ast
 from dataclasses import is_dataclass, fields
 from typing import get_args, get_origin, List, Optional, Union, Any, ForwardRef
 import typing

ifex/model/ifex_generator.py → models/ifex/ifex_generator.py

@@ -19,7 +19,7 @@
 
 # For other features from parser module
 from typing import Any
-from ifex.templates import JinjaTemplateEnv
+from output_filters.templates import JinjaTemplateEnv
 
 # Module global - probably soon to be modified to run-time instantiation of a
 # JinjaTemplateEnv instance instead.
@@ -109,7 +109,7 @@ def gen_dict_with_template_file(variables : dict, templatefile):
 # ----------------------------------------------------------------
 # MAIN = Standalone test Code only, not normal use.
 import sys
-from ifex.model.ifex_parser import get_ast_from_yaml_file
+from models.ifex.ifex_parser import get_ast_from_yaml_file
 if __name__ == '__main__':
     yaml_file = sys.argv[1]
     template_dir = sys.argv[2]

ifex/model/ifex_parser.py → models/ifex/ifex_parser.py

@@ -15,7 +15,7 @@
 
 import yaml, dacite
 from typing import Dict, Any
-from ifex.model.ifex_ast import AST
+from models.ifex.ifex_ast import AST
 
 
 def read_yaml_file(filename) -> str:

ifex/model/type_checking_constructor_mixin.py → models/ifex/type_checking_constructor_mixin.py

@@ -7,7 +7,7 @@
 
 from dataclasses import dataclass, fields
 from typing import get_type_hints, List, Optional, Any
-from ifex.model.ifex_ast_introspect import is_list, actual_type, inner_type, is_optional, field_is_optional, is_any
+from models.ifex.ifex_ast_introspect import is_list, actual_type, inner_type, is_optional, field_is_optional, is_any
 
 def is_correct_type(value, _type):
     if type(value) is list and is_list(_type):

other/protobuf/README.md → models/protobuf/README.md

@@ -8,12 +8,14 @@
 - `protobuf_lark.py` - Input parser, creates Protobuf AST
 - `protobuf_ast.py` - Dataclass definitions for a Protobuf AST
 - `protobuf_ast_construction.py` - Define helper functions that build AST nodes
-- `protobuf_to_ifex.py` - AST Model-to-model transformation Protobuf->IFEX
 
 ## How to run
 
-For simple conversions or tests, the protobuf_to_ifex.py has a main method and can be run as a script directly. It will print out the 
-result as IFEX Core IDL in YAML text:
+Go to input_filters/protobuf directory to find the converter.
+
+For simple conversions or tests, the protobuf_to_ifex.py has a main method and
+can be run as a script directly. It will print out the result as IFEX Core IDL
+in YAML text:
 
 ```
 python protobuf_to_ifex.py <input.proto>

other/protobuf/protobuf_ast_construction.py → models/protobuf/protobuf_ast_construction.py

@@ -12,8 +12,8 @@
 # is and why.  It follows the exact same pattern.
 
 from dataclasses import is_dataclass
-from ifex.model.type_checking_constructor_mixin import add_constructor
-import other.protobuf.protobuf_ast as protobuf_ast
+from models.ifex.type_checking_constructor_mixin import add_constructor
+import models.protobuf.protobuf_ast as protobuf_ast
 
 def add_constructors_to_protobuf_ast_model() -> None:
     """ Mix-in the type-checking constructor support into each of the protobuf_ast classes: """

other/protobuf/protobuf_lark.py → models/protobuf/protobuf_lark.py

@@ -4,7 +4,7 @@
 # This file is part of the IFEX project
 
 from lark import Lark, logger, Tree, Token
-from other.protobuf.protobuf_ast import Option, EnumField, Enumeration, MapField, Field, Import, Message, RPC, Service, Proto
+from models.protobuf.protobuf_ast import Option, EnumField, Enumeration, MapField, Field, Import, Message, RPC, Service, Proto
 import lark
 import re
 import sys
@@ -39,7 +39,7 @@
 
 
 # Use protobuf_construction mixin
-import other.protobuf.protobuf_ast_construction as protobuf_ast_construction
+import models.protobuf.protobuf_ast_construction as protobuf_ast_construction
 protobuf_ast_construction.add_constructors_to_protobuf_ast_model()
 
 # Remove lines matching regexp