NAS-130495 / 24.10 / Add support for deploying custom apps (#14167)

* Rename ix-chart to ix-app

* Fix app.available

* Add basic custom app svc

* Call app.custom.create if custom app creation is desired

* Add basic validation for app.custom.create

* Expect serialized yaml file for docker compose

* Allow specifying both string/dict for custom compose config

* Modify setup install app util to account for custom app

* Amend updating config func to account for custom apps

* Amend metadata func to account for custom app

* Complete custom app create implementation:

* Add validation util for custom compose app

* Get updates to work for custom apps

* Fix upgrade available flag for custom apps

* Add internal endpoint for deleting app

* Add util to get rendered docker compose config

* Add ability to convert any app to advanced app

* Add public endpoint to convert any app to custom app

* Fix minor bug

* Fix max length

src/middlewared/middlewared/plugins/apps/crud.py

@@ -4,12 +4,15 @@
 import shutil
 import textwrap
 
-from middlewared.schema import accepts, Bool, Dict, Int, List, returns, Str
-from middlewared.service import CallError, CRUDService, filterable, InstanceNotFound, job, pass_app, private
+from middlewared.schema import accepts, Bool, Dict, Int, List, Ref, returns, Str
+from middlewared.service import (
+    CallError, CRUDService, filterable, InstanceNotFound, job, pass_app, private, ValidationErrors
+)
 from middlewared.utils import filter_list
 from middlewared.validators import Match, Range
 
 from .compose_utils import compose_action
+from .custom_app_utils import validate_payload
 from .ix_apps.lifecycle import add_context_to_values, get_current_app_config, update_app_config
 from .ix_apps.metadata import update_app_metadata, update_app_metadata_for_portals
 from .ix_apps.path import get_installed_app_path, get_installed_app_version_path
@@ -116,11 +119,23 @@ def config(self, app_name):
         app = self.get_instance__sync(app_name)
         return get_current_app_config(app_name, app['version'])
 
+    @accepts(Str('app_name'), roles=['APPS_WRITE'])
+    @returns(Ref('app_query'))
+    @job(lock=lambda args: f'app_start_{args[0]}')
+    async def convert_to_custom(self, job, app_name):
+        """
+        Convert `app_name` to a custom app.
+        """
+        return await self.middleware.call('app.custom.convert', job, app_name)
+
     @accepts(
         Dict(
             'app_create',
+            Bool('custom_app', default=False),
             Dict('values', additional_attrs=True, private=True),
-            Str('catalog_app', required=True),
+            Dict('custom_compose_config', additional_attrs=True, private=True),
+            Str('custom_compose_config_string', private=True, max_length=2**31),
+            Str('catalog_app', required=False),
             Str(
                 'app_name', required=True, validators=[Match(
                     r'^[a-z]([-a-z0-9]*[a-z0-9])?$',
@@ -151,6 +166,14 @@ def do_create(self, job, data):
         if self.middleware.call_sync('app.query', [['id', '=', data['app_name']]]):
             raise CallError(f'Application with name {data["app_name"]} already exists', errno=errno.EEXIST)
 
+        if data['custom_app']:
+            return self.middleware.call_sync('app.custom.create', data, job)
+
+        verrors = ValidationErrors()
+        if not data.get('catalog_app'):
+            verrors.add('app_create.catalog_app', 'This field is required')
+        verrors.check()
+
         app_name = data['app_name']
         complete_app_details = self.middleware.call_sync('catalog.get_app_details', data['catalog_app'], {
             'train': data['train'],
@@ -218,6 +241,8 @@ def create_internal(
         Dict(
             'app_update',
             Dict('values', additional_attrs=True, private=True),
+            Dict('custom_compose_config', additional_attrs=True, private=True),
+            Str('custom_compose_config_string', private=True, max_length=2**31),
         )
     )
     @job(lock=lambda args: f'app_update_{args[0]}')
@@ -232,25 +257,33 @@ def do_update(self, job, app_name, data):
     @private
     def update_internal(self, job, app, data, progress_keyword='Update'):
         app_name = app['id']
-        config = get_current_app_config(app_name, app['version'])
-        config.update(data['values'])
-        # We use update=False because we want defaults to be populated again if they are not present in the payload
-        # Why this is not dangerous is because the defaults will be added only if they are not present/configured for
-        # the app in question
-        app_version_details = self.middleware.call_sync(
-            'catalog.app_version_details', get_installed_app_version_path(app_name, app['version'])
-        )
+        if app['custom_app']:
+            if progress_keyword == 'Update':
+                new_values = validate_payload(data, 'app_update')
+            else:
+                new_values = get_current_app_config(app_name, app['version'])
+        else:
+            config = get_current_app_config(app_name, app['version'])
+            config.update(data['values'])
+            # We use update=False because we want defaults to be populated again if they are not present in the payload
+            # Why this is not dangerous is because the defaults will be added only if they are not present/configured
+            # for the app in question
+            app_version_details = self.middleware.call_sync(
+                'catalog.app_version_details', get_installed_app_version_path(app_name, app['version'])
+            )
 
-        new_values = self.middleware.call_sync(
-            'app.schema.normalize_and_validate_values', app_version_details, config, True,
-            get_installed_app_path(app_name), app
-        )
+            new_values = self.middleware.call_sync(
+                'app.schema.normalize_and_validate_values', app_version_details, config, True,
+                get_installed_app_path(app_name), app
+            )
+            new_values = add_context_to_values(app_name, new_values, app['metadata'], update=True)
 
         job.set_progress(25, 'Initial Validation completed')
 
-        new_values = add_context_to_values(app_name, new_values, app['metadata'], update=True)
-        update_app_config(app_name, app['version'], new_values)
-        update_app_metadata_for_portals(app_name, app['version'])
+        update_app_config(app_name, app['version'], new_values, custom_app=app['custom_app'])
+        if app['custom_app'] is False:
+            # TODO: Eventually we would want this to be executed for custom apps as well
+            update_app_metadata_for_portals(app_name, app['version'])
         job.set_progress(60, 'Configuration updated, updating docker resources')
         compose_action(app_name, app['version'], 'up', force_recreate=True, remove_orphans=True)
 
@@ -271,6 +304,10 @@ def do_delete(self, job, app_name, options):
         Delete `app_name` app.
         """
         app_config = self.get_instance__sync(app_name)
+        return self.delete_internal(job, app_name, app_config, options)
+
+    @private
+    def delete_internal(self, job, app_name, app_config, options):
         job.set_progress(20, f'Deleting {app_name!r} app')
         compose_action(
             app_name, app_config['version'], 'down', remove_orphans=True,

src/middlewared/middlewared/plugins/apps/custom_app.py

@@ -0,0 +1,85 @@
+import contextlib
+import shutil
+
+from catalog_reader.custom_app import get_version_details
+
+from middlewared.service import CallError, Service
+
+from .compose_utils import compose_action
+from .custom_app_utils import validate_payload
+from .ix_apps.lifecycle import get_rendered_template_config_of_app, update_app_config
+from .ix_apps.metadata import update_app_metadata
+from .ix_apps.path import get_installed_app_path
+from .ix_apps.setup import setup_install_app_dir
+
+
+class AppCustomService(Service):
+
+    class Config:
+        namespace = 'app.custom'
+        private = True
+
+    def convert(self, job, app_name):
+        app = self.middleware.call_sync('app.get_instance', app_name)
+        if app['custom_app'] is True:
+            raise CallError(f'{app_name!r} is already a custom app')
+
+        rendered_config = get_rendered_template_config_of_app(app_name, app['version'])
+        if not rendered_config:
+            raise CallError(f'No rendered config found for {app_name!r}')
+
+        job.set_progress(10, 'Completed initial validation for conversion of app to custom app')
+        # What needs to happen here is the following:
+        # Merge all available compose files into one of the app and hold on to it
+        # Do an uninstall of the app and create it again with the new compose file
+        # Update metadata to reflect that this is a custom app
+        # Finally update collective metadata
+        job.set_progress(20, 'Removing existing app\'s docker resources')
+        self.middleware.call_sync(
+            'app.delete_internal', type('dummy_job', (object,), {'set_progress': lambda *args: None})(),
+            app_name, app, {'remove_images': False, 'remove_ix_volumes': False}
+        )
+
+        return self.create({
+            'app_name': app_name,
+            'custom_compose_config': rendered_config,
+        }, job)
+
+    def create(self, data, job=None, progress_base=0):
+        """
+        Create a custom app.
+        """
+        compose_config = validate_payload(data, 'app_create')
+
+        def update_progress(percentage_done, message):
+            job.set_progress(int((100 - progress_base) * (percentage_done / 100)) + progress_base, message)
+
+        # For debug purposes
+        job = job or type('dummy_job', (object,), {'set_progress': lambda *args: None})()
+        update_progress(25, 'Initial validation completed for custom app creation')
+
+        app_name = data['app_name']
+        app_version_details = get_version_details()
+        version = app_version_details['version']
+        try:
+            update_progress(35, 'Setting up App directory')
+            setup_install_app_dir(app_name, app_version_details, custom_app=True)
+            update_app_config(app_name, version, compose_config, custom_app=True)
+            update_app_metadata(app_name, app_version_details, migrated=False, custom_app=True)
+
+            update_progress(60, 'App installation in progress, pulling images')
+            compose_action(app_name, version, 'up', force_recreate=True, remove_orphans=True)
+        except Exception as e:
+            update_progress(80, f'Failure occurred while installing {app_name!r}, cleaning up')
+            for method, args, kwargs in (
+                (compose_action, (app_name, version, 'down'), {'remove_orphans': True}),
+                (shutil.rmtree, (get_installed_app_path(app_name),), {}),
+            ):
+                with contextlib.suppress(Exception):
+                    method(*args, **kwargs)
+
+            raise e from None
+        else:
+            self.middleware.call_sync('app.metadata.generate').wait_sync(raise_error=True)
+            job.set_progress(100, f'{app_name!r} installed successfully')
+            return self.middleware.call_sync('app.get_instance', app_name)

src/middlewared/middlewared/plugins/apps/custom_app_utils.py

@@ -0,0 +1,23 @@
+import yaml
+
+from middlewared.service import ValidationErrors
+
+
+def validate_payload(data: dict, schema: str) -> dict:
+    verrors = ValidationErrors()
+    compose_keys = ('custom_compose_config', 'custom_compose_config_string')
+    if all(not data.get(k) for k in compose_keys):
+        verrors.add(f'{schema}.custom_compose_config', 'This field is required')
+    elif all(data.get(k) for k in compose_keys):
+        verrors.add(f'{schema}.custom_compose_config_string', 'Only one of these fields should be provided')
+
+    compose_config = data.get('custom_compose_config')
+    if data.get('custom_compose_config_string'):
+        try:
+            compose_config = yaml.safe_load(data['custom_compose_config_string'])
+        except yaml.YAMLError:
+            verrors.add('app_create.custom_compose_config_string', 'Invalid YAML provided')
+
+    verrors.check()
+
+    return compose_config

src/middlewared/middlewared/plugins/apps/ix_apps/lifecycle.py

@@ -1,4 +1,5 @@
 import copy
+import contextlib
 import pathlib
 import typing
 import yaml
@@ -7,10 +8,21 @@
 
 from .path import (
     get_installed_app_config_path, get_installed_app_rendered_dir_path, get_installed_app_version_path,
+    get_installed_custom_app_compose_file,
 )
 from .utils import CONTEXT_KEY_NAME, run
 
 
+def get_rendered_template_config_of_app(app_name: str, version: str) -> dict:
+    rendered_config = {}
+    for rendered_file in get_rendered_templates_of_app(app_name, version):
+        with contextlib.suppress(FileNotFoundError, yaml.YAMLError):
+            with open(rendered_file, 'r') as f:
+                rendered_config.update(yaml.safe_load(f.read()))
+
+    return rendered_config
+
+
 def get_rendered_templates_of_app(app_name: str, version: str) -> list[str]:
     result = []
     for entry in pathlib.Path(get_installed_app_rendered_dir_path(app_name, version)).iterdir():
@@ -36,11 +48,15 @@ def render_compose_templates(app_version_path: str, values_file_path: str):
         raise CallError(f'Failed to render compose templates: {cp.stderr}')
 
 
-def update_app_config(app_name: str, version: str, values: dict[str, typing.Any]) -> None:
+def update_app_config(app_name: str, version: str, values: dict[str, typing.Any], custom_app: bool = False) -> None:
     write_new_app_config(app_name, version, values)
-    render_compose_templates(
-        get_installed_app_version_path(app_name, version), get_installed_app_config_path(app_name, version)
-    )
+    if custom_app:
+        with open(get_installed_custom_app_compose_file(app_name, version), 'w') as f:
+            f.write(yaml.safe_dump(values))
+    else:
+        render_compose_templates(
+            get_installed_app_version_path(app_name, version), get_installed_app_config_path(app_name, version)
+        )
 
 
 def get_action_context(app_name: str) -> dict[str, typing.Any]:

src/middlewared/middlewared/plugins/apps/ix_apps/metadata.py

@@ -15,14 +15,18 @@ def get_app_metadata(app_name: str) -> dict[str, typing.Any]:
         return {}
 
 
-def update_app_metadata(app_name: str, app_version_details: dict, migrated: bool | None = None):
+def update_app_metadata(
+    app_name: str, app_version_details: dict, migrated: bool | None = None, custom_app: bool = False,
+):
     migrated = get_app_metadata(app_name).get('migrated', False) if migrated is None else migrated
     with open(get_installed_app_metadata_path(app_name), 'w') as f:
         f.write(yaml.safe_dump({
             'metadata': app_version_details['app_metadata'],
             'migrated': migrated,
+            'custom_app': custom_app,
             **{k: app_version_details[k] for k in ('version', 'human_version')},
             **get_portals_and_app_notes(app_name, app_version_details['version']),
+            # TODO: We should not try to get portals for custom apps for now
         }))
 
 

src/middlewared/middlewared/plugins/apps/ix_apps/path.py

@@ -47,5 +47,9 @@ def get_installed_app_config_path(app_name: str, version: str) -> str:
     return os.path.join(get_installed_app_version_path(app_name, version), 'user_config.yaml')
 
 
+def get_installed_custom_app_compose_file(app_name: str, version: str) -> str:
+    return os.path.join(get_installed_app_rendered_dir_path(app_name, version), 'docker-compose.yaml')
+
+
 def get_installed_app_rendered_dir_path(app_name: str, version: str) -> str:
     return os.path.join(get_installed_app_version_path(app_name, version), 'templates/rendered')

src/middlewared/middlewared/plugins/apps/ix_apps/portals.py

@@ -1,23 +1,14 @@
-import contextlib
-
-import yaml
-
 from apps_validation.portals import IX_NOTES_KEY, IX_PORTAL_KEY, validate_portals_and_notes, ValidationErrors
 
-from .lifecycle import get_rendered_templates_of_app
+from .lifecycle import get_rendered_template_config_of_app
 
 
 def normalized_port_value(scheme: str, port: int) -> str:
     return '' if ((scheme == 'http' and port == 80) or (scheme == 'https' and port == 443)) else f':{port}'
 
 
 def get_portals_and_app_notes(app_name: str, version: str) -> dict:
-    rendered_config = {}
-    for rendered_file in get_rendered_templates_of_app(app_name, version):
-        with contextlib.suppress(FileNotFoundError, yaml.YAMLError):
-            with open(rendered_file, 'r') as f:
-                rendered_config.update(yaml.safe_load(f.read()))
-
+    rendered_config = get_rendered_template_config_of_app(app_name, version)
     portal_and_notes_config = {
         k: rendered_config[k]
         for k in (IX_NOTES_KEY, IX_PORTAL_KEY)

src/middlewared/middlewared/plugins/apps/ix_apps/query.py

@@ -26,7 +26,11 @@ def __hash__(self):
 def upgrade_available_for_app(
     version_mapping: dict[str, dict[str, dict[str, str]]], app_metadata: dict
 ) -> bool:
-    if version_mapping.get(app_metadata['train'], {}).get(app_metadata['name']):
+    # TODO: Eventually we would want this to work as well but this will always require middleware changes
+    #  depending on what new functionality we want introduced for custom app, so let's take care of this at that point
+    if (app_metadata['name'] == 'custom-app' and app_metadata['train'] == 'stable') is False and version_mapping.get(
+        app_metadata['train'], {}
+    ).get(app_metadata['name']):
         return parse_version(app_metadata['version']) < parse_version(
             version_mapping[app_metadata['train']][app_metadata['name']]['version']
         )

src/middlewared/middlewared/plugins/apps/ix_apps/setup.py

@@ -1,14 +1,29 @@
 import os
 import shutil
+import textwrap
+import yaml
 
 from .metadata import update_app_yaml_for_last_update
 from .path import get_app_parent_config_path, get_installed_app_version_path
 
 
-def setup_install_app_dir(app_name: str, app_version_details: dict):
+def setup_install_app_dir(app_name: str, app_version_details: dict, custom_app: bool = False):
     os.makedirs(os.path.join(get_app_parent_config_path(), app_name, 'versions'), exist_ok=True)
     to_install_app_version = os.path.basename(app_version_details['version'])
     destination = get_installed_app_version_path(app_name, to_install_app_version)
-    shutil.copytree(app_version_details['location'], destination)
+    if custom_app:
+        # TODO: See if it makes sense to creat a dummy app on apps side instead
+        os.makedirs(os.path.join(destination, 'templates/rendered'), exist_ok=True)
+        with open(os.path.join(destination, 'README.md'), 'w') as f:
+            f.write(textwrap.dedent('''
+            # Custom App
+
+            This is a custom app where user can use his/her own docker compose file for deploying services.
+            '''))
+
+        with open(os.path.join(destination, 'app.yaml'), 'w') as f:
+            f.write(yaml.safe_dump(app_version_details['app_metadata']))
+    else:
+        shutil.copytree(app_version_details['location'], destination)
 
     update_app_yaml_for_last_update(destination, app_version_details['last_update'])