From 2dace0d27466909a958e0a82def9e2cfb929add5 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Thu, 8 Aug 2024 09:06:58 +0500
Subject: [PATCH 01/20] Rename ix-chart to ix-app

---
 src/middlewared/middlewared/plugins/catalog/apps.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/middlewared/middlewared/plugins/catalog/apps.py b/src/middlewared/middlewared/plugins/catalog/apps.py
index 3b3e8f875991b..03de7ff692162 100644
--- a/src/middlewared/middlewared/plugins/catalog/apps.py
+++ b/src/middlewared/middlewared/plugins/catalog/apps.py
@@ -17,7 +17,7 @@ async def latest(self, filters, options):
         return filter_list(
             await self.middleware.call(
                 'app.available', [
-                    ['last_update', '!=', None], ['name', '!=', 'ix-chart'],
+                    ['last_update', '!=', None], ['name', '!=', 'ix-app'],
                 ], {'order_by': ['-last_update']}
             ), filters, options
         )

From e00faeb7261038f32863c212e3ac47b7e5dc2120 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Thu, 8 Aug 2024 09:09:16 +0500
Subject: [PATCH 02/20] Fix app.available

---
 src/middlewared/middlewared/plugins/catalog/apps.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/middlewared/middlewared/plugins/catalog/apps.py b/src/middlewared/middlewared/plugins/catalog/apps.py
index 03de7ff692162..4c236ab7166fe 100644
--- a/src/middlewared/middlewared/plugins/catalog/apps.py
+++ b/src/middlewared/middlewared/plugins/catalog/apps.py
@@ -58,8 +58,8 @@ def available(self, filters, options):
 
         results = []
         installed_apps = [
-            (app['chart_metadata']['name'], app['catalog'], app['catalog_train'])
-            for app in []
+            (app['metadata']['name'], app['metadata']['train'])
+            for app in self.middleware.call_sync('app.query')
         ]
 
         catalog = self.middleware.call_sync('catalog.config')
@@ -70,7 +70,7 @@ def available(self, filters, options):
             for app_data in train_data.values():
                 results.append({
                     'catalog': catalog['label'],
-                    'installed': (app_data['name'], catalog['label'], train) in installed_apps,
+                    'installed': (app_data['name'], train) in installed_apps,
                     'train': train,
                     **app_data,
                 })

From bfa4e4070e66dd402a82b18e6e922ea6af172411 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Thu, 8 Aug 2024 09:49:36 +0500
Subject: [PATCH 03/20] Add basic custom app svc

---
 src/middlewared/middlewared/plugins/apps/custom_app.py | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 src/middlewared/middlewared/plugins/apps/custom_app.py

diff --git a/src/middlewared/middlewared/plugins/apps/custom_app.py b/src/middlewared/middlewared/plugins/apps/custom_app.py
new file mode 100644
index 0000000000000..f97b203891073
--- /dev/null
+++ b/src/middlewared/middlewared/plugins/apps/custom_app.py
@@ -0,0 +1,8 @@
+from middlewared.service import Service
+
+
+class AppCustomService(Service):
+
+    class Config:
+        namespace = 'app.custom'
+        private = True

From f2187fb14f818b1ab1bfdf499225320248e5e1ed Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Thu, 8 Aug 2024 09:57:09 +0500
Subject: [PATCH 04/20] Call app.custom.create if custom app creation is
 desired

---
 .../middlewared/plugins/apps/crud.py            | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/middlewared/middlewared/plugins/apps/crud.py b/src/middlewared/middlewared/plugins/apps/crud.py
index 2e37ae56d0dd7..6339d990441e8 100644
--- a/src/middlewared/middlewared/plugins/apps/crud.py
+++ b/src/middlewared/middlewared/plugins/apps/crud.py
@@ -5,7 +5,9 @@
 import textwrap
 
 from middlewared.schema import accepts, Bool, Dict, Int, List, returns, Str
-from middlewared.service import CallError, CRUDService, filterable, InstanceNotFound, job, pass_app, private
+from middlewared.service import (
+    CallError, CRUDService, filterable, InstanceNotFound, job, pass_app, private, ValidationErrors
+)
 from middlewared.utils import filter_list
 from middlewared.validators import Match, Range
 
@@ -119,8 +121,10 @@ def config(self, app_name):
     @accepts(
         Dict(
             'app_create',
+            Bool('custom_app', default=False),
             Dict('values', additional_attrs=True, private=True),
-            Str('catalog_app', required=True),
+            Str('custom_compose', null=True),
+            Str('catalog_app', required=False),
             Str(
                 'app_name', required=True, validators=[Match(
                     r'^[a-z]([-a-z0-9]*[a-z0-9])?$',
@@ -151,6 +155,15 @@ def do_create(self, job, data):
         if self.middleware.call_sync('app.query', [['id', '=', data['app_name']]]):
             raise CallError(f'Application with name {data["app_name"]} already exists', errno=errno.EEXIST)
 
+        if data['custom_app']:
+            self.middleware.call_sync('app.custom.create', data, job)
+            return
+
+        verrors = ValidationErrors()
+        if not data.get('catalog_app'):
+            verrors.add('app_create.catalog_app', 'This field is required')
+        verrors.check()
+
         app_name = data['app_name']
         complete_app_details = self.middleware.call_sync('catalog.get_app_details', data['catalog_app'], {
             'train': data['train'],

From b329c1e63b4b64ac9b3b924e8f0a25ce0ffad011 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Thu, 8 Aug 2024 10:01:12 +0500
Subject: [PATCH 05/20] Add basic validation for app.custom.create

---
 .../middlewared/plugins/apps/custom_app.py     | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/src/middlewared/middlewared/plugins/apps/custom_app.py b/src/middlewared/middlewared/plugins/apps/custom_app.py
index f97b203891073..4c52dee7b9158 100644
--- a/src/middlewared/middlewared/plugins/apps/custom_app.py
+++ b/src/middlewared/middlewared/plugins/apps/custom_app.py
@@ -1,4 +1,4 @@
-from middlewared.service import Service
+from middlewared.service import Service, ValidationErrors
 
 
 class AppCustomService(Service):
@@ -6,3 +6,19 @@ class AppCustomService(Service):
     class Config:
         namespace = 'app.custom'
         private = True
+
+    def create(self, data, job=None):
+        """
+        Create a custom app.
+        """
+        verrors = ValidationErrors()
+        if not data.get('custom_compose'):
+            verrors.add('app_create.custom_compose', 'This field is required')
+
+        verrors.check()
+
+        # For debug purposes
+        job = job or type('dummy_job', (object,), {'set_progress': lambda *args: None})()
+
+        app_name = data['app_name']
+        compose_contents = data['custom_compose']

From b780c73c96b2308bb08876d350b5c90f67ae1c52 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Thu, 8 Aug 2024 10:05:00 +0500
Subject: [PATCH 06/20] Expect serialized yaml file for docker compose

---
 src/middlewared/middlewared/plugins/apps/crud.py       | 2 +-
 src/middlewared/middlewared/plugins/apps/custom_app.py | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/middlewared/middlewared/plugins/apps/crud.py b/src/middlewared/middlewared/plugins/apps/crud.py
index 6339d990441e8..12ca19f4385f1 100644
--- a/src/middlewared/middlewared/plugins/apps/crud.py
+++ b/src/middlewared/middlewared/plugins/apps/crud.py
@@ -123,7 +123,7 @@ def config(self, app_name):
             'app_create',
             Bool('custom_app', default=False),
             Dict('values', additional_attrs=True, private=True),
-            Str('custom_compose', null=True),
+            Dict('custom_compose_config', additional_attrs=True, private=True),
             Str('catalog_app', required=False),
             Str(
                 'app_name', required=True, validators=[Match(
diff --git a/src/middlewared/middlewared/plugins/apps/custom_app.py b/src/middlewared/middlewared/plugins/apps/custom_app.py
index 4c52dee7b9158..c571b6d4929da 100644
--- a/src/middlewared/middlewared/plugins/apps/custom_app.py
+++ b/src/middlewared/middlewared/plugins/apps/custom_app.py
@@ -12,13 +12,14 @@ def create(self, data, job=None):
         Create a custom app.
         """
         verrors = ValidationErrors()
-        if not data.get('custom_compose'):
-            verrors.add('app_create.custom_compose', 'This field is required')
+        if not data.get('custom_compose_config'):
+            verrors.add('app_create.custom_compose_config', 'This field is required')
 
         verrors.check()
 
         # For debug purposes
         job = job or type('dummy_job', (object,), {'set_progress': lambda *args: None})()
+        job.set_progress(25, 'Initial validation completed for custom app creation')
 
         app_name = data['app_name']
         compose_contents = data['custom_compose']

From 3c6509e34683128c3008e80c9e45bbd8ace6a95c Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Thu, 8 Aug 2024 10:30:52 +0500
Subject: [PATCH 07/20] Allow specifying both string/dict for custom compose
 config

---
 src/middlewared/middlewared/plugins/apps/crud.py   |  1 +
 .../middlewared/plugins/apps/custom_app.py         | 14 +++++++++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/middlewared/middlewared/plugins/apps/crud.py b/src/middlewared/middlewared/plugins/apps/crud.py
index 12ca19f4385f1..669fd85d2d230 100644
--- a/src/middlewared/middlewared/plugins/apps/crud.py
+++ b/src/middlewared/middlewared/plugins/apps/crud.py
@@ -124,6 +124,7 @@ def config(self, app_name):
             Bool('custom_app', default=False),
             Dict('values', additional_attrs=True, private=True),
             Dict('custom_compose_config', additional_attrs=True, private=True),
+            Str('custom_compose_config_string', private=True),
             Str('catalog_app', required=False),
             Str(
                 'app_name', required=True, validators=[Match(
diff --git a/src/middlewared/middlewared/plugins/apps/custom_app.py b/src/middlewared/middlewared/plugins/apps/custom_app.py
index c571b6d4929da..21a24ddb60117 100644
--- a/src/middlewared/middlewared/plugins/apps/custom_app.py
+++ b/src/middlewared/middlewared/plugins/apps/custom_app.py
@@ -1,3 +1,5 @@
+import yaml
+
 from middlewared.service import Service, ValidationErrors
 
 
@@ -12,8 +14,18 @@ def create(self, data, job=None):
         Create a custom app.
         """
         verrors = ValidationErrors()
-        if not data.get('custom_compose_config'):
+        compose_keys = ('custom_compose_config', 'custom_compose_config_string')
+        if all(not data.get(k) for k in compose_keys):
             verrors.add('app_create.custom_compose_config', 'This field is required')
+        elif all(data.get(k) for k in compose_keys):
+            verrors.add('app_create.custom_compose_config_string', 'Only one of these fields should be provided')
+
+        compose_config = data.get('custom_compose_config')
+        if data.get('custom_compose_config_string'):
+            try:
+                compose_config = yaml.YAMLError(data['custom_compose_config_string'])
+            except yaml.YAMLError:
+                verrors.add('app_create.custom_compose_config_string', 'Invalid YAML provided')
 
         verrors.check()
 

From f26ef2259d767e0aceafbd177b07ee747aec2974 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Thu, 8 Aug 2024 10:56:04 +0500
Subject: [PATCH 08/20] Modify setup install app util to account for custom app

---
 .../middlewared/plugins/apps/ix_apps/setup.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/src/middlewared/middlewared/plugins/apps/ix_apps/setup.py b/src/middlewared/middlewared/plugins/apps/ix_apps/setup.py
index b595d06a752cb..b7f91981a9c76 100644
--- a/src/middlewared/middlewared/plugins/apps/ix_apps/setup.py
+++ b/src/middlewared/middlewared/plugins/apps/ix_apps/setup.py
@@ -1,14 +1,29 @@
 import os
 import shutil
+import textwrap
+import yaml
 
 from .metadata import update_app_yaml_for_last_update
 from .path import get_app_parent_config_path, get_installed_app_version_path
 
 
-def setup_install_app_dir(app_name: str, app_version_details: dict):
+def setup_install_app_dir(app_name: str, app_version_details: dict, custom_app: bool = False):
     os.makedirs(os.path.join(get_app_parent_config_path(), app_name, 'versions'), exist_ok=True)
     to_install_app_version = os.path.basename(app_version_details['version'])
     destination = get_installed_app_version_path(app_name, to_install_app_version)
-    shutil.copytree(app_version_details['location'], destination)
+    if custom_app:
+        # TODO: See if it makes sense to creat a dummy app on apps side instead
+        os.makedirs(os.path.join(destination, 'templates/rendered'), exist_ok=True)
+        with open(os.path.join(destination, 'README.md'), 'w') as f:
+            f.write(textwrap.dedent('''
+            # Custom App
+
+            This is a custom app where user can use his/her own docker compose file for deploying services.
+            '''))
+
+        with open(os.path.join(destination, 'app.yaml'), 'w') as f:
+            f.write(yaml.safe_dump(app_version_details['app_metadata']))
+    else:
+        shutil.copytree(app_version_details['location'], destination)
 
     update_app_yaml_for_last_update(destination, app_version_details['last_update'])

From b50a18fb6fd403e0bc0b4e2f87f7f4546e57bd48 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Thu, 8 Aug 2024 11:06:36 +0500
Subject: [PATCH 09/20] Amend updating config func to account for custom apps

---
 .../middlewared/plugins/apps/ix_apps/lifecycle.py   | 13 +++++++++----
 .../middlewared/plugins/apps/ix_apps/path.py        |  4 ++++
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/middlewared/middlewared/plugins/apps/ix_apps/lifecycle.py b/src/middlewared/middlewared/plugins/apps/ix_apps/lifecycle.py
index ed91c48962f6e..7eb0ebc404238 100644
--- a/src/middlewared/middlewared/plugins/apps/ix_apps/lifecycle.py
+++ b/src/middlewared/middlewared/plugins/apps/ix_apps/lifecycle.py
@@ -7,6 +7,7 @@
 
 from .path import (
     get_installed_app_config_path, get_installed_app_rendered_dir_path, get_installed_app_version_path,
+    get_installed_custom_app_compose_file,
 )
 from .utils import CONTEXT_KEY_NAME, run
 
@@ -36,11 +37,15 @@ def render_compose_templates(app_version_path: str, values_file_path: str):
         raise CallError(f'Failed to render compose templates: {cp.stderr}')
 
 
-def update_app_config(app_name: str, version: str, values: dict[str, typing.Any]) -> None:
+def update_app_config(app_name: str, version: str, values: dict[str, typing.Any], custom_app: bool = False) -> None:
     write_new_app_config(app_name, version, values)
-    render_compose_templates(
-        get_installed_app_version_path(app_name, version), get_installed_app_config_path(app_name, version)
-    )
+    if custom_app:
+        with open(get_installed_custom_app_compose_file(app_name, version), 'w') as f:
+            f.write(yaml.safe_dump(values))
+    else:
+        render_compose_templates(
+            get_installed_app_version_path(app_name, version), get_installed_app_config_path(app_name, version)
+        )
 
 
 def get_action_context(app_name: str) -> dict[str, typing.Any]:
diff --git a/src/middlewared/middlewared/plugins/apps/ix_apps/path.py b/src/middlewared/middlewared/plugins/apps/ix_apps/path.py
index effd7003271d0..d0f0f73854cd0 100644
--- a/src/middlewared/middlewared/plugins/apps/ix_apps/path.py
+++ b/src/middlewared/middlewared/plugins/apps/ix_apps/path.py
@@ -47,5 +47,9 @@ def get_installed_app_config_path(app_name: str, version: str) -> str:
     return os.path.join(get_installed_app_version_path(app_name, version), 'user_config.yaml')
 
 
+def get_installed_custom_app_compose_file(app_name: str, version: str) -> str:
+    return os.path.join(get_installed_app_rendered_dir_path(app_name, version), 'docker-compose.yaml')
+
+
 def get_installed_app_rendered_dir_path(app_name: str, version: str) -> str:
     return os.path.join(get_installed_app_version_path(app_name, version), 'templates/rendered')

From e727113cacba6e54c78f46f85b892af10b8d49da Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Thu, 8 Aug 2024 11:22:10 +0500
Subject: [PATCH 10/20] Amend metadata func to account for custom app

---
 src/middlewared/middlewared/plugins/apps/ix_apps/metadata.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/middlewared/middlewared/plugins/apps/ix_apps/metadata.py b/src/middlewared/middlewared/plugins/apps/ix_apps/metadata.py
index c87f97fad68b7..de9dc12514c5d 100644
--- a/src/middlewared/middlewared/plugins/apps/ix_apps/metadata.py
+++ b/src/middlewared/middlewared/plugins/apps/ix_apps/metadata.py
@@ -15,12 +15,15 @@ def get_app_metadata(app_name: str) -> dict[str, typing.Any]:
         return {}
 
 
-def update_app_metadata(app_name: str, app_version_details: dict, migrated: bool | None = None):
+def update_app_metadata(
+    app_name: str, app_version_details: dict, migrated: bool | None = None, custom_app: bool = False,
+):
     migrated = get_app_metadata(app_name).get('migrated', False) if migrated is None else migrated
     with open(get_installed_app_metadata_path(app_name), 'w') as f:
         f.write(yaml.safe_dump({
             'metadata': app_version_details['app_metadata'],
             'migrated': migrated,
+            'custom_app': custom_app,
             **{k: app_version_details[k] for k in ('version', 'human_version')},
             **get_portals_and_app_notes(app_name, app_version_details['version']),
         }))

From 814a2512515946702185fbb31769ac12d4599886 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Thu, 8 Aug 2024 11:33:40 +0500
Subject: [PATCH 11/20] Complete custom app create implementation:

---
 .../middlewared/plugins/apps/custom_app.py    | 35 ++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/src/middlewared/middlewared/plugins/apps/custom_app.py b/src/middlewared/middlewared/plugins/apps/custom_app.py
index 21a24ddb60117..6d3b6d0e598ff 100644
--- a/src/middlewared/middlewared/plugins/apps/custom_app.py
+++ b/src/middlewared/middlewared/plugins/apps/custom_app.py
@@ -1,7 +1,17 @@
+import contextlib
+import shutil
 import yaml
 
+from catalog_reader.custom_app import get_version_details
+
 from middlewared.service import Service, ValidationErrors
 
+from .compose_utils import compose_action
+from .ix_apps.lifecycle import update_app_config
+from .ix_apps.metadata import update_app_metadata
+from .ix_apps.path import get_installed_app_path
+from .ix_apps.setup import setup_install_app_dir
+
 
 class AppCustomService(Service):
 
@@ -34,4 +44,27 @@ def create(self, data, job=None):
         job.set_progress(25, 'Initial validation completed for custom app creation')
 
         app_name = data['app_name']
-        compose_contents = data['custom_compose']
+        app_version_details = get_version_details()
+        version = app_version_details['version']
+        try:
+            job.set_progress(35, 'Setting up App directory')
+            setup_install_app_dir(app_name, app_version_details)
+            update_app_config(app_name, version, compose_config)
+            update_app_metadata(app_name, app_version_details, migrated=False, custom_app=True)
+
+            job.set_progress(60, 'App installation in progress, pulling images')
+            compose_action(app_name, version, 'up', force_recreate=True, remove_orphans=True)
+        except Exception as e:
+            job.set_progress(80, f'Failure occurred while installing {app_name!r}, cleaning up')
+            for method, args, kwargs in (
+                (compose_action, (app_name, version, 'down'), {'remove_orphans': True}),
+                (shutil.rmtree, (get_installed_app_path(app_name),), {}),
+            ):
+                with contextlib.suppress(Exception):
+                    method(*args, **kwargs)
+
+            raise e from None
+        else:
+            self.middleware.call_sync('app.metadata.generate').wait_sync(raise_error=True)
+            job.set_progress(100, f'{app_name!r} installed successfully')
+            return self.middleware.call_sync('app.get_instance', app_name)

From c2724fb18255a4ed297bc4c38b3e959fa1718706 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Thu, 8 Aug 2024 12:13:28 +0500
Subject: [PATCH 12/20] Add validation util for custom compose app

---
 .../middlewared/plugins/apps/custom_app.py    | 17 ++------------
 .../plugins/apps/custom_app_utils.py          | 23 +++++++++++++++++++
 2 files changed, 25 insertions(+), 15 deletions(-)
 create mode 100644 src/middlewared/middlewared/plugins/apps/custom_app_utils.py

diff --git a/src/middlewared/middlewared/plugins/apps/custom_app.py b/src/middlewared/middlewared/plugins/apps/custom_app.py
index 6d3b6d0e598ff..e1c6f961943d4 100644
--- a/src/middlewared/middlewared/plugins/apps/custom_app.py
+++ b/src/middlewared/middlewared/plugins/apps/custom_app.py
@@ -7,6 +7,7 @@
 from middlewared.service import Service, ValidationErrors
 
 from .compose_utils import compose_action
+from .custom_app_utils import validate_payload
 from .ix_apps.lifecycle import update_app_config
 from .ix_apps.metadata import update_app_metadata
 from .ix_apps.path import get_installed_app_path
@@ -23,21 +24,7 @@ def create(self, data, job=None):
         """
         Create a custom app.
         """
-        verrors = ValidationErrors()
-        compose_keys = ('custom_compose_config', 'custom_compose_config_string')
-        if all(not data.get(k) for k in compose_keys):
-            verrors.add('app_create.custom_compose_config', 'This field is required')
-        elif all(data.get(k) for k in compose_keys):
-            verrors.add('app_create.custom_compose_config_string', 'Only one of these fields should be provided')
-
-        compose_config = data.get('custom_compose_config')
-        if data.get('custom_compose_config_string'):
-            try:
-                compose_config = yaml.YAMLError(data['custom_compose_config_string'])
-            except yaml.YAMLError:
-                verrors.add('app_create.custom_compose_config_string', 'Invalid YAML provided')
-
-        verrors.check()
+        compose_config = validate_payload(data, 'app_create')
 
         # For debug purposes
         job = job or type('dummy_job', (object,), {'set_progress': lambda *args: None})()
diff --git a/src/middlewared/middlewared/plugins/apps/custom_app_utils.py b/src/middlewared/middlewared/plugins/apps/custom_app_utils.py
new file mode 100644
index 0000000000000..906778384a10b
--- /dev/null
+++ b/src/middlewared/middlewared/plugins/apps/custom_app_utils.py
@@ -0,0 +1,23 @@
+import yaml
+
+from middlewared.service import ValidationErrors
+
+
+def validate_payload(data: dict, schema: str) -> dict:
+    verrors = ValidationErrors()
+    compose_keys = ('custom_compose_config', 'custom_compose_config_string')
+    if all(not data.get(k) for k in compose_keys):
+        verrors.add(f'{schema}.custom_compose_config', 'This field is required')
+    elif all(data.get(k) for k in compose_keys):
+        verrors.add(f'{schema}.custom_compose_config_string', 'Only one of these fields should be provided')
+
+    compose_config = data.get('custom_compose_config')
+    if data.get('custom_compose_config_string'):
+        try:
+            compose_config = yaml.YAMLError(data['custom_compose_config_string'])
+        except yaml.YAMLError:
+            verrors.add('app_create.custom_compose_config_string', 'Invalid YAML provided')
+
+    verrors.check()
+
+    return compose_config

From 36ea6d7b04e84c29bccdfccb12e2b3c4be973792 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Thu, 8 Aug 2024 12:41:49 +0500
Subject: [PATCH 13/20] Get updates to work for custom apps

---
 .../middlewared/plugins/apps/crud.py          | 44 ++++++++++++-------
 .../middlewared/plugins/apps/custom_app.py    |  3 +-
 .../plugins/apps/ix_apps/metadata.py          |  1 +
 3 files changed, 29 insertions(+), 19 deletions(-)

diff --git a/src/middlewared/middlewared/plugins/apps/crud.py b/src/middlewared/middlewared/plugins/apps/crud.py
index 669fd85d2d230..d26e85c82f348 100644
--- a/src/middlewared/middlewared/plugins/apps/crud.py
+++ b/src/middlewared/middlewared/plugins/apps/crud.py
@@ -12,6 +12,7 @@
 from middlewared.validators import Match, Range
 
 from .compose_utils import compose_action
+from .custom_app_utils import validate_payload
 from .ix_apps.lifecycle import add_context_to_values, get_current_app_config, update_app_config
 from .ix_apps.metadata import update_app_metadata, update_app_metadata_for_portals
 from .ix_apps.path import get_installed_app_path, get_installed_app_version_path
@@ -157,8 +158,7 @@ def do_create(self, job, data):
             raise CallError(f'Application with name {data["app_name"]} already exists', errno=errno.EEXIST)
 
         if data['custom_app']:
-            self.middleware.call_sync('app.custom.create', data, job)
-            return
+            return self.middleware.call_sync('app.custom.create', data, job)
 
         verrors = ValidationErrors()
         if not data.get('catalog_app'):
@@ -232,6 +232,8 @@ def create_internal(
         Dict(
             'app_update',
             Dict('values', additional_attrs=True, private=True),
+            Dict('custom_compose_config', additional_attrs=True, private=True),
+            Str('custom_compose_config_string', private=True),
         )
     )
     @job(lock=lambda args: f'app_update_{args[0]}')
@@ -246,25 +248,33 @@ def do_update(self, job, app_name, data):
     @private
     def update_internal(self, job, app, data, progress_keyword='Update'):
         app_name = app['id']
-        config = get_current_app_config(app_name, app['version'])
-        config.update(data['values'])
-        # We use update=False because we want defaults to be populated again if they are not present in the payload
-        # Why this is not dangerous is because the defaults will be added only if they are not present/configured for
-        # the app in question
-        app_version_details = self.middleware.call_sync(
-            'catalog.app_version_details', get_installed_app_version_path(app_name, app['version'])
-        )
+        if app['custom_app']:
+            if progress_keyword == 'Update':
+                new_values = validate_payload(data, 'app_update')
+            else:
+                new_values = get_current_app_config(app_name, app['version'])
+        else:
+            config = get_current_app_config(app_name, app['version'])
+            config.update(data['values'])
+            # We use update=False because we want defaults to be populated again if they are not present in the payload
+            # Why this is not dangerous is because the defaults will be added only if they are not present/configured
+            # for the app in question
+            app_version_details = self.middleware.call_sync(
+                'catalog.app_version_details', get_installed_app_version_path(app_name, app['version'])
+            )
 
-        new_values = self.middleware.call_sync(
-            'app.schema.normalize_and_validate_values', app_version_details, config, True,
-            get_installed_app_path(app_name), app
-        )
+            new_values = self.middleware.call_sync(
+                'app.schema.normalize_and_validate_values', app_version_details, config, True,
+                get_installed_app_path(app_name), app
+            )
+            new_values = add_context_to_values(app_name, new_values, app['metadata'], update=True)
 
         job.set_progress(25, 'Initial Validation completed')
 
-        new_values = add_context_to_values(app_name, new_values, app['metadata'], update=True)
-        update_app_config(app_name, app['version'], new_values)
-        update_app_metadata_for_portals(app_name, app['version'])
+        update_app_config(app_name, app['version'], new_values, custom_app=app['custom_app'])
+        if app['custom_app'] is False:
+            # TODO: Eventually we would want this to be executed for custom apps as well
+            update_app_metadata_for_portals(app_name, app['version'])
         job.set_progress(60, 'Configuration updated, updating docker resources')
         compose_action(app_name, app['version'], 'up', force_recreate=True, remove_orphans=True)
 
diff --git a/src/middlewared/middlewared/plugins/apps/custom_app.py b/src/middlewared/middlewared/plugins/apps/custom_app.py
index e1c6f961943d4..e3ca02089c780 100644
--- a/src/middlewared/middlewared/plugins/apps/custom_app.py
+++ b/src/middlewared/middlewared/plugins/apps/custom_app.py
@@ -1,10 +1,9 @@
 import contextlib
 import shutil
-import yaml
 
 from catalog_reader.custom_app import get_version_details
 
-from middlewared.service import Service, ValidationErrors
+from middlewared.service import Service
 
 from .compose_utils import compose_action
 from .custom_app_utils import validate_payload
diff --git a/src/middlewared/middlewared/plugins/apps/ix_apps/metadata.py b/src/middlewared/middlewared/plugins/apps/ix_apps/metadata.py
index de9dc12514c5d..45be0da61f012 100644
--- a/src/middlewared/middlewared/plugins/apps/ix_apps/metadata.py
+++ b/src/middlewared/middlewared/plugins/apps/ix_apps/metadata.py
@@ -26,6 +26,7 @@ def update_app_metadata(
             'custom_app': custom_app,
             **{k: app_version_details[k] for k in ('version', 'human_version')},
             **get_portals_and_app_notes(app_name, app_version_details['version']),
+            # TODO: We should not try to get portals for custom apps for now
         }))
 
 

From 7b057af7943b491893d588aef2d1fb4521a828cb Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Thu, 8 Aug 2024 12:45:35 +0500
Subject: [PATCH 14/20] Fix upgrade available flag for custom apps

---
 src/middlewared/middlewared/plugins/apps/ix_apps/query.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/middlewared/middlewared/plugins/apps/ix_apps/query.py b/src/middlewared/middlewared/plugins/apps/ix_apps/query.py
index c20c78ff8a6d5..6930a4c4e514e 100644
--- a/src/middlewared/middlewared/plugins/apps/ix_apps/query.py
+++ b/src/middlewared/middlewared/plugins/apps/ix_apps/query.py
@@ -26,7 +26,11 @@ def __hash__(self):
 def upgrade_available_for_app(
     version_mapping: dict[str, dict[str, dict[str, str]]], app_metadata: dict
 ) -> bool:
-    if version_mapping.get(app_metadata['train'], {}).get(app_metadata['name']):
+    # TODO: Eventually we would want this to work as well but this will always require middleware changes
+    #  depending on what new functionality we want introduced for custom app, so let's take care of this at that point
+    if (app_metadata['name'] == 'custom-app' and app_metadata['train'] == 'stable') is False and version_mapping.get(
+        app_metadata['train'], {}
+    ).get(app_metadata['name']):
         return parse_version(app_metadata['version']) < parse_version(
             version_mapping[app_metadata['train']][app_metadata['name']]['version']
         )

From cf52649aa4fce6a321d63e4bd49928caef500464 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Thu, 8 Aug 2024 13:06:51 +0500
Subject: [PATCH 15/20] Add internal endpoint for deleting app

---
 src/middlewared/middlewared/plugins/apps/crud.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/middlewared/middlewared/plugins/apps/crud.py b/src/middlewared/middlewared/plugins/apps/crud.py
index d26e85c82f348..73ba67b41c950 100644
--- a/src/middlewared/middlewared/plugins/apps/crud.py
+++ b/src/middlewared/middlewared/plugins/apps/crud.py
@@ -295,6 +295,10 @@ def do_delete(self, job, app_name, options):
         Delete `app_name` app.
         """
         app_config = self.get_instance__sync(app_name)
+        return self.delete_internal(job, app_name, app_config, options)
+
+    @private
+    def delete_internal(self, job, app_name, app_config, options):
         job.set_progress(20, f'Deleting {app_name!r} app')
         compose_action(
             app_name, app_config['version'], 'down', remove_orphans=True,

From d37fd6663a58aaf30fab692707c4cbd21458c1af Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Thu, 8 Aug 2024 13:14:25 +0500
Subject: [PATCH 16/20] Add util to get rendered docker compose config

---
 .../middlewared/plugins/apps/ix_apps/lifecycle.py   | 11 +++++++++++
 .../middlewared/plugins/apps/ix_apps/portals.py     | 13 ++-----------
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/middlewared/middlewared/plugins/apps/ix_apps/lifecycle.py b/src/middlewared/middlewared/plugins/apps/ix_apps/lifecycle.py
index 7eb0ebc404238..ac177f88691f1 100644
--- a/src/middlewared/middlewared/plugins/apps/ix_apps/lifecycle.py
+++ b/src/middlewared/middlewared/plugins/apps/ix_apps/lifecycle.py
@@ -1,4 +1,5 @@
 import copy
+import contextlib
 import pathlib
 import typing
 import yaml
@@ -12,6 +13,16 @@
 from .utils import CONTEXT_KEY_NAME, run
 
 
+def get_rendered_template_config_of_app(app_name: str, version: str) -> dict:
+    rendered_config = {}
+    for rendered_file in get_rendered_templates_of_app(app_name, version):
+        with contextlib.suppress(FileNotFoundError, yaml.YAMLError):
+            with open(rendered_file, 'r') as f:
+                rendered_config.update(yaml.safe_load(f.read()))
+
+    return rendered_config
+
+
 def get_rendered_templates_of_app(app_name: str, version: str) -> list[str]:
     result = []
     for entry in pathlib.Path(get_installed_app_rendered_dir_path(app_name, version)).iterdir():
diff --git a/src/middlewared/middlewared/plugins/apps/ix_apps/portals.py b/src/middlewared/middlewared/plugins/apps/ix_apps/portals.py
index db34de6b4a2dd..5a547ab9867b7 100644
--- a/src/middlewared/middlewared/plugins/apps/ix_apps/portals.py
+++ b/src/middlewared/middlewared/plugins/apps/ix_apps/portals.py
@@ -1,10 +1,6 @@
-import contextlib
-
-import yaml
-
 from apps_validation.portals import IX_NOTES_KEY, IX_PORTAL_KEY, validate_portals_and_notes, ValidationErrors
 
-from .lifecycle import get_rendered_templates_of_app
+from .lifecycle import get_rendered_template_config_of_app
 
 
 def normalized_port_value(scheme: str, port: int) -> str:
@@ -12,12 +8,7 @@ def normalized_port_value(scheme: str, port: int) -> str:
 
 
 def get_portals_and_app_notes(app_name: str, version: str) -> dict:
-    rendered_config = {}
-    for rendered_file in get_rendered_templates_of_app(app_name, version):
-        with contextlib.suppress(FileNotFoundError, yaml.YAMLError):
-            with open(rendered_file, 'r') as f:
-                rendered_config.update(yaml.safe_load(f.read()))
-
+    rendered_config = get_rendered_template_config_of_app(app_name, version)
     portal_and_notes_config = {
         k: rendered_config[k]
         for k in (IX_NOTES_KEY, IX_PORTAL_KEY)

From dff79357088acf4b4ea2f267d1cbb6f7e7ecea04 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Thu, 8 Aug 2024 13:29:09 +0500
Subject: [PATCH 17/20] Add ability to convert any app to advanced app

---
 .../middlewared/plugins/apps/custom_app.py    | 43 ++++++++++++++++---
 1 file changed, 36 insertions(+), 7 deletions(-)

diff --git a/src/middlewared/middlewared/plugins/apps/custom_app.py b/src/middlewared/middlewared/plugins/apps/custom_app.py
index e3ca02089c780..e867c6ff31f5b 100644
--- a/src/middlewared/middlewared/plugins/apps/custom_app.py
+++ b/src/middlewared/middlewared/plugins/apps/custom_app.py
@@ -3,11 +3,11 @@
 
 from catalog_reader.custom_app import get_version_details
 
-from middlewared.service import Service
+from middlewared.service import CallError, Service
 
 from .compose_utils import compose_action
 from .custom_app_utils import validate_payload
-from .ix_apps.lifecycle import update_app_config
+from .ix_apps.lifecycle import get_rendered_template_config_of_app, update_app_config
 from .ix_apps.metadata import update_app_metadata
 from .ix_apps.path import get_installed_app_path
 from .ix_apps.setup import setup_install_app_dir
@@ -19,29 +19,58 @@ class Config:
         namespace = 'app.custom'
         private = True
 
-    def create(self, data, job=None):
+    def convert(self, job, app_name):
+        app = self.middleware.call_sync('app.get_instance', app_name)
+        if app['custom_app'] is True:
+            raise CallError(f'{app_name!r} is already a custom app')
+
+        rendered_config = get_rendered_template_config_of_app(app_name, app['version'])
+        if not rendered_config:
+            raise CallError(f'No rendered config found for {app_name!r}')
+
+        job.set_progress(10, 'Completed initial validation for conversion of app to custom app')
+        # What needs to happen here is the following:
+        # Merge all available compose files into one of the app and hold on to it
+        # Do an uninstall of the app and create it again with the new compose file
+        # Update metadata to reflect that this is a custom app
+        # Finally update collective metadata
+        job.set_progress(20, 'Removing existing app\'s docker resources')
+        self.middleware.call_sync(
+            'app.delete_internal', type('dummy_job', (object,), {'set_progress': lambda *args: None})(),
+            app_name, app, {'remove_images': False, 'remove_ix_volumes': False}
+        )
+
+        return self.create({
+            'app_name': app_name,
+            'custom_compose_config': rendered_config,
+        }, job)
+
+    def create(self, data, job=None, progress_base=0):
         """
         Create a custom app.
         """
         compose_config = validate_payload(data, 'app_create')
 
+        def update_progress(percentage_done, message):
+            job.set_progress(int((100 - progress_base) * (percentage_done / 100)) + progress_base, message)
+
         # For debug purposes
         job = job or type('dummy_job', (object,), {'set_progress': lambda *args: None})()
-        job.set_progress(25, 'Initial validation completed for custom app creation')
+        update_progress(25, 'Initial validation completed for custom app creation')
 
         app_name = data['app_name']
         app_version_details = get_version_details()
         version = app_version_details['version']
         try:
-            job.set_progress(35, 'Setting up App directory')
+            update_progress(35, 'Setting up App directory')
             setup_install_app_dir(app_name, app_version_details)
             update_app_config(app_name, version, compose_config)
             update_app_metadata(app_name, app_version_details, migrated=False, custom_app=True)
 
-            job.set_progress(60, 'App installation in progress, pulling images')
+            update_progress(60, 'App installation in progress, pulling images')
             compose_action(app_name, version, 'up', force_recreate=True, remove_orphans=True)
         except Exception as e:
-            job.set_progress(80, f'Failure occurred while installing {app_name!r}, cleaning up')
+            update_progress(80, f'Failure occurred while installing {app_name!r}, cleaning up')
             for method, args, kwargs in (
                 (compose_action, (app_name, version, 'down'), {'remove_orphans': True}),
                 (shutil.rmtree, (get_installed_app_path(app_name),), {}),

From 459ed7831c53186619ae23038951934a50f99bd0 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Thu, 8 Aug 2024 13:43:07 +0500
Subject: [PATCH 18/20] Add public endpoint to convert any app to custom app

---
 src/middlewared/middlewared/plugins/apps/crud.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/middlewared/middlewared/plugins/apps/crud.py b/src/middlewared/middlewared/plugins/apps/crud.py
index 73ba67b41c950..261e967c4ab40 100644
--- a/src/middlewared/middlewared/plugins/apps/crud.py
+++ b/src/middlewared/middlewared/plugins/apps/crud.py
@@ -4,7 +4,7 @@
 import shutil
 import textwrap
 
-from middlewared.schema import accepts, Bool, Dict, Int, List, returns, Str
+from middlewared.schema import accepts, Bool, Dict, Int, List, Ref, returns, Str
 from middlewared.service import (
     CallError, CRUDService, filterable, InstanceNotFound, job, pass_app, private, ValidationErrors
 )
@@ -119,6 +119,15 @@ def config(self, app_name):
         app = self.get_instance__sync(app_name)
         return get_current_app_config(app_name, app['version'])
 
+    @accepts(Str('app_name'), roles=['APPS_WRITE'])
+    @returns(Ref('app_query'))
+    @job(lock=lambda args: f'app_start_{args[0]}')
+    async def convert_to_custom(self, job, app_name):
+        """
+        Convert `app_name` to a custom app.
+        """
+        return await self.middleware.call('app.custom.convert', job, app_name)
+
     @accepts(
         Dict(
             'app_create',

From cd2636e87f6e118491ba9b9a7e146e1ae45adc0a Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Thu, 8 Aug 2024 20:07:34 +0500
Subject: [PATCH 19/20] Fix minor bug

---
 src/middlewared/middlewared/plugins/apps/custom_app.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/middlewared/middlewared/plugins/apps/custom_app.py b/src/middlewared/middlewared/plugins/apps/custom_app.py
index e867c6ff31f5b..442a177be9dac 100644
--- a/src/middlewared/middlewared/plugins/apps/custom_app.py
+++ b/src/middlewared/middlewared/plugins/apps/custom_app.py
@@ -63,8 +63,8 @@ def update_progress(percentage_done, message):
         version = app_version_details['version']
         try:
             update_progress(35, 'Setting up App directory')
-            setup_install_app_dir(app_name, app_version_details)
-            update_app_config(app_name, version, compose_config)
+            setup_install_app_dir(app_name, app_version_details, custom_app=True)
+            update_app_config(app_name, version, compose_config, custom_app=True)
             update_app_metadata(app_name, app_version_details, migrated=False, custom_app=True)
 
             update_progress(60, 'App installation in progress, pulling images')

From e36895f0ad809b8f264db065424e2329a659b4a4 Mon Sep 17 00:00:00 2001
From: Waqar Ahmed <waqarahmedjoyia@live.com>
Date: Fri, 9 Aug 2024 09:12:02 +0500
Subject: [PATCH 20/20] Fix max length

---
 src/middlewared/middlewared/plugins/apps/crud.py             | 4 ++--
 src/middlewared/middlewared/plugins/apps/custom_app_utils.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/middlewared/middlewared/plugins/apps/crud.py b/src/middlewared/middlewared/plugins/apps/crud.py
index 261e967c4ab40..6b0cbf5510617 100644
--- a/src/middlewared/middlewared/plugins/apps/crud.py
+++ b/src/middlewared/middlewared/plugins/apps/crud.py
@@ -134,7 +134,7 @@ async def convert_to_custom(self, job, app_name):
             Bool('custom_app', default=False),
             Dict('values', additional_attrs=True, private=True),
             Dict('custom_compose_config', additional_attrs=True, private=True),
-            Str('custom_compose_config_string', private=True),
+            Str('custom_compose_config_string', private=True, max_length=2**31),
             Str('catalog_app', required=False),
             Str(
                 'app_name', required=True, validators=[Match(
@@ -242,7 +242,7 @@ def create_internal(
             'app_update',
             Dict('values', additional_attrs=True, private=True),
             Dict('custom_compose_config', additional_attrs=True, private=True),
-            Str('custom_compose_config_string', private=True),
+            Str('custom_compose_config_string', private=True, max_length=2**31),
         )
     )
     @job(lock=lambda args: f'app_update_{args[0]}')
diff --git a/src/middlewared/middlewared/plugins/apps/custom_app_utils.py b/src/middlewared/middlewared/plugins/apps/custom_app_utils.py
index 906778384a10b..a81666e2195fa 100644
--- a/src/middlewared/middlewared/plugins/apps/custom_app_utils.py
+++ b/src/middlewared/middlewared/plugins/apps/custom_app_utils.py
@@ -14,7 +14,7 @@ def validate_payload(data: dict, schema: str) -> dict:
     compose_config = data.get('custom_compose_config')
     if data.get('custom_compose_config_string'):
         try:
-            compose_config = yaml.YAMLError(data['custom_compose_config_string'])
+            compose_config = yaml.safe_load(data['custom_compose_config_string'])
         except yaml.YAMLError:
             verrors.add('app_create.custom_compose_config_string', 'Invalid YAML provided')