From 3696d960271a17a71bc877b662ef49cd76bcdecb Mon Sep 17 00:00:00 2001 From: sonicaj Date: Fri, 9 Aug 2024 09:20:35 +0500 Subject: [PATCH] NAS-130495 / 24.10 / Add support for deploying custom apps (#14167) * Rename ix-chart to ix-app * Fix app.available * Add basic custom app svc * Call app.custom.create if custom app creation is desired * Add basic validation for app.custom.create * Expect serialized yaml file for docker compose * Allow specifying both string/dict for custom compose config * Modify setup install app util to account for custom app * Amend updating config func to account for custom apps * Amend metadata func to account for custom app * Complete custom app create implementation: * Add validation util for custom compose app * Get updates to work for custom apps * Fix upgrade available flag for custom apps * Add internal endpoint for deleting app * Add util to get rendered docker compose config * Add ability to convert any app to advanced app * Add public endpoint to convert any app to custom app * Fix minor bug * Fix max length --- .../middlewared/plugins/apps/crud.py | 73 ++++++++++++---- .../middlewared/plugins/apps/custom_app.py | 85 +++++++++++++++++++ .../plugins/apps/custom_app_utils.py | 23 +++++ .../plugins/apps/ix_apps/lifecycle.py | 24 +++++- .../plugins/apps/ix_apps/metadata.py | 6 +- .../middlewared/plugins/apps/ix_apps/path.py | 4 + .../plugins/apps/ix_apps/portals.py | 13 +-- .../middlewared/plugins/apps/ix_apps/query.py | 6 +- .../middlewared/plugins/apps/ix_apps/setup.py | 19 ++++- .../middlewared/plugins/catalog/apps.py | 8 +- 10 files changed, 220 insertions(+), 41 deletions(-) create mode 100644 src/middlewared/middlewared/plugins/apps/custom_app.py create mode 100644 src/middlewared/middlewared/plugins/apps/custom_app_utils.py diff --git a/src/middlewared/middlewared/plugins/apps/crud.py b/src/middlewared/middlewared/plugins/apps/crud.py index 2e37ae56d0dd7..6b0cbf5510617 100644 --- a/src/middlewared/middlewared/plugins/apps/crud.py +++ b/src/middlewared/middlewared/plugins/apps/crud.py @@ -4,12 +4,15 @@ import shutil import textwrap -from middlewared.schema import accepts, Bool, Dict, Int, List, returns, Str -from middlewared.service import CallError, CRUDService, filterable, InstanceNotFound, job, pass_app, private +from middlewared.schema import accepts, Bool, Dict, Int, List, Ref, returns, Str +from middlewared.service import ( + CallError, CRUDService, filterable, InstanceNotFound, job, pass_app, private, ValidationErrors +) from middlewared.utils import filter_list from middlewared.validators import Match, Range from .compose_utils import compose_action +from .custom_app_utils import validate_payload from .ix_apps.lifecycle import add_context_to_values, get_current_app_config, update_app_config from .ix_apps.metadata import update_app_metadata, update_app_metadata_for_portals from .ix_apps.path import get_installed_app_path, get_installed_app_version_path @@ -116,11 +119,23 @@ def config(self, app_name): app = self.get_instance__sync(app_name) return get_current_app_config(app_name, app['version']) + @accepts(Str('app_name'), roles=['APPS_WRITE']) + @returns(Ref('app_query')) + @job(lock=lambda args: f'app_start_{args[0]}') + async def convert_to_custom(self, job, app_name): + """ + Convert `app_name` to a custom app. + """ + return await self.middleware.call('app.custom.convert', job, app_name) + @accepts( Dict( 'app_create', + Bool('custom_app', default=False), Dict('values', additional_attrs=True, private=True), - Str('catalog_app', required=True), + Dict('custom_compose_config', additional_attrs=True, private=True), + Str('custom_compose_config_string', private=True, max_length=2**31), + Str('catalog_app', required=False), Str( 'app_name', required=True, validators=[Match( r'^[a-z]([-a-z0-9]*[a-z0-9])?$', @@ -151,6 +166,14 @@ def do_create(self, job, data): if self.middleware.call_sync('app.query', [['id', '=', data['app_name']]]): raise CallError(f'Application with name {data["app_name"]} already exists', errno=errno.EEXIST) + if data['custom_app']: + return self.middleware.call_sync('app.custom.create', data, job) + + verrors = ValidationErrors() + if not data.get('catalog_app'): + verrors.add('app_create.catalog_app', 'This field is required') + verrors.check() + app_name = data['app_name'] complete_app_details = self.middleware.call_sync('catalog.get_app_details', data['catalog_app'], { 'train': data['train'], @@ -218,6 +241,8 @@ def create_internal( Dict( 'app_update', Dict('values', additional_attrs=True, private=True), + Dict('custom_compose_config', additional_attrs=True, private=True), + Str('custom_compose_config_string', private=True, max_length=2**31), ) ) @job(lock=lambda args: f'app_update_{args[0]}') @@ -232,25 +257,33 @@ def do_update(self, job, app_name, data): @private def update_internal(self, job, app, data, progress_keyword='Update'): app_name = app['id'] - config = get_current_app_config(app_name, app['version']) - config.update(data['values']) - # We use update=False because we want defaults to be populated again if they are not present in the payload - # Why this is not dangerous is because the defaults will be added only if they are not present/configured for - # the app in question - app_version_details = self.middleware.call_sync( - 'catalog.app_version_details', get_installed_app_version_path(app_name, app['version']) - ) + if app['custom_app']: + if progress_keyword == 'Update': + new_values = validate_payload(data, 'app_update') + else: + new_values = get_current_app_config(app_name, app['version']) + else: + config = get_current_app_config(app_name, app['version']) + config.update(data['values']) + # We use update=False because we want defaults to be populated again if they are not present in the payload + # Why this is not dangerous is because the defaults will be added only if they are not present/configured + # for the app in question + app_version_details = self.middleware.call_sync( + 'catalog.app_version_details', get_installed_app_version_path(app_name, app['version']) + ) - new_values = self.middleware.call_sync( - 'app.schema.normalize_and_validate_values', app_version_details, config, True, - get_installed_app_path(app_name), app - ) + new_values = self.middleware.call_sync( + 'app.schema.normalize_and_validate_values', app_version_details, config, True, + get_installed_app_path(app_name), app + ) + new_values = add_context_to_values(app_name, new_values, app['metadata'], update=True) job.set_progress(25, 'Initial Validation completed') - new_values = add_context_to_values(app_name, new_values, app['metadata'], update=True) - update_app_config(app_name, app['version'], new_values) - update_app_metadata_for_portals(app_name, app['version']) + update_app_config(app_name, app['version'], new_values, custom_app=app['custom_app']) + if app['custom_app'] is False: + # TODO: Eventually we would want this to be executed for custom apps as well + update_app_metadata_for_portals(app_name, app['version']) job.set_progress(60, 'Configuration updated, updating docker resources') compose_action(app_name, app['version'], 'up', force_recreate=True, remove_orphans=True) @@ -271,6 +304,10 @@ def do_delete(self, job, app_name, options): Delete `app_name` app. """ app_config = self.get_instance__sync(app_name) + return self.delete_internal(job, app_name, app_config, options) + + @private + def delete_internal(self, job, app_name, app_config, options): job.set_progress(20, f'Deleting {app_name!r} app') compose_action( app_name, app_config['version'], 'down', remove_orphans=True, diff --git a/src/middlewared/middlewared/plugins/apps/custom_app.py b/src/middlewared/middlewared/plugins/apps/custom_app.py new file mode 100644 index 0000000000000..442a177be9dac --- /dev/null +++ b/src/middlewared/middlewared/plugins/apps/custom_app.py @@ -0,0 +1,85 @@ +import contextlib +import shutil + +from catalog_reader.custom_app import get_version_details + +from middlewared.service import CallError, Service + +from .compose_utils import compose_action +from .custom_app_utils import validate_payload +from .ix_apps.lifecycle import get_rendered_template_config_of_app, update_app_config +from .ix_apps.metadata import update_app_metadata +from .ix_apps.path import get_installed_app_path +from .ix_apps.setup import setup_install_app_dir + + +class AppCustomService(Service): + + class Config: + namespace = 'app.custom' + private = True + + def convert(self, job, app_name): + app = self.middleware.call_sync('app.get_instance', app_name) + if app['custom_app'] is True: + raise CallError(f'{app_name!r} is already a custom app') + + rendered_config = get_rendered_template_config_of_app(app_name, app['version']) + if not rendered_config: + raise CallError(f'No rendered config found for {app_name!r}') + + job.set_progress(10, 'Completed initial validation for conversion of app to custom app') + # What needs to happen here is the following: + # Merge all available compose files into one of the app and hold on to it + # Do an uninstall of the app and create it again with the new compose file + # Update metadata to reflect that this is a custom app + # Finally update collective metadata + job.set_progress(20, 'Removing existing app\'s docker resources') + self.middleware.call_sync( + 'app.delete_internal', type('dummy_job', (object,), {'set_progress': lambda *args: None})(), + app_name, app, {'remove_images': False, 'remove_ix_volumes': False} + ) + + return self.create({ + 'app_name': app_name, + 'custom_compose_config': rendered_config, + }, job) + + def create(self, data, job=None, progress_base=0): + """ + Create a custom app. + """ + compose_config = validate_payload(data, 'app_create') + + def update_progress(percentage_done, message): + job.set_progress(int((100 - progress_base) * (percentage_done / 100)) + progress_base, message) + + # For debug purposes + job = job or type('dummy_job', (object,), {'set_progress': lambda *args: None})() + update_progress(25, 'Initial validation completed for custom app creation') + + app_name = data['app_name'] + app_version_details = get_version_details() + version = app_version_details['version'] + try: + update_progress(35, 'Setting up App directory') + setup_install_app_dir(app_name, app_version_details, custom_app=True) + update_app_config(app_name, version, compose_config, custom_app=True) + update_app_metadata(app_name, app_version_details, migrated=False, custom_app=True) + + update_progress(60, 'App installation in progress, pulling images') + compose_action(app_name, version, 'up', force_recreate=True, remove_orphans=True) + except Exception as e: + update_progress(80, f'Failure occurred while installing {app_name!r}, cleaning up') + for method, args, kwargs in ( + (compose_action, (app_name, version, 'down'), {'remove_orphans': True}), + (shutil.rmtree, (get_installed_app_path(app_name),), {}), + ): + with contextlib.suppress(Exception): + method(*args, **kwargs) + + raise e from None + else: + self.middleware.call_sync('app.metadata.generate').wait_sync(raise_error=True) + job.set_progress(100, f'{app_name!r} installed successfully') + return self.middleware.call_sync('app.get_instance', app_name) diff --git a/src/middlewared/middlewared/plugins/apps/custom_app_utils.py b/src/middlewared/middlewared/plugins/apps/custom_app_utils.py new file mode 100644 index 0000000000000..a81666e2195fa --- /dev/null +++ b/src/middlewared/middlewared/plugins/apps/custom_app_utils.py @@ -0,0 +1,23 @@ +import yaml + +from middlewared.service import ValidationErrors + + +def validate_payload(data: dict, schema: str) -> dict: + verrors = ValidationErrors() + compose_keys = ('custom_compose_config', 'custom_compose_config_string') + if all(not data.get(k) for k in compose_keys): + verrors.add(f'{schema}.custom_compose_config', 'This field is required') + elif all(data.get(k) for k in compose_keys): + verrors.add(f'{schema}.custom_compose_config_string', 'Only one of these fields should be provided') + + compose_config = data.get('custom_compose_config') + if data.get('custom_compose_config_string'): + try: + compose_config = yaml.safe_load(data['custom_compose_config_string']) + except yaml.YAMLError: + verrors.add('app_create.custom_compose_config_string', 'Invalid YAML provided') + + verrors.check() + + return compose_config diff --git a/src/middlewared/middlewared/plugins/apps/ix_apps/lifecycle.py b/src/middlewared/middlewared/plugins/apps/ix_apps/lifecycle.py index ed91c48962f6e..ac177f88691f1 100644 --- a/src/middlewared/middlewared/plugins/apps/ix_apps/lifecycle.py +++ b/src/middlewared/middlewared/plugins/apps/ix_apps/lifecycle.py @@ -1,4 +1,5 @@ import copy +import contextlib import pathlib import typing import yaml @@ -7,10 +8,21 @@ from .path import ( get_installed_app_config_path, get_installed_app_rendered_dir_path, get_installed_app_version_path, + get_installed_custom_app_compose_file, ) from .utils import CONTEXT_KEY_NAME, run +def get_rendered_template_config_of_app(app_name: str, version: str) -> dict: + rendered_config = {} + for rendered_file in get_rendered_templates_of_app(app_name, version): + with contextlib.suppress(FileNotFoundError, yaml.YAMLError): + with open(rendered_file, 'r') as f: + rendered_config.update(yaml.safe_load(f.read())) + + return rendered_config + + def get_rendered_templates_of_app(app_name: str, version: str) -> list[str]: result = [] for entry in pathlib.Path(get_installed_app_rendered_dir_path(app_name, version)).iterdir(): @@ -36,11 +48,15 @@ def render_compose_templates(app_version_path: str, values_file_path: str): raise CallError(f'Failed to render compose templates: {cp.stderr}') -def update_app_config(app_name: str, version: str, values: dict[str, typing.Any]) -> None: +def update_app_config(app_name: str, version: str, values: dict[str, typing.Any], custom_app: bool = False) -> None: write_new_app_config(app_name, version, values) - render_compose_templates( - get_installed_app_version_path(app_name, version), get_installed_app_config_path(app_name, version) - ) + if custom_app: + with open(get_installed_custom_app_compose_file(app_name, version), 'w') as f: + f.write(yaml.safe_dump(values)) + else: + render_compose_templates( + get_installed_app_version_path(app_name, version), get_installed_app_config_path(app_name, version) + ) def get_action_context(app_name: str) -> dict[str, typing.Any]: diff --git a/src/middlewared/middlewared/plugins/apps/ix_apps/metadata.py b/src/middlewared/middlewared/plugins/apps/ix_apps/metadata.py index c87f97fad68b7..45be0da61f012 100644 --- a/src/middlewared/middlewared/plugins/apps/ix_apps/metadata.py +++ b/src/middlewared/middlewared/plugins/apps/ix_apps/metadata.py @@ -15,14 +15,18 @@ def get_app_metadata(app_name: str) -> dict[str, typing.Any]: return {} -def update_app_metadata(app_name: str, app_version_details: dict, migrated: bool | None = None): +def update_app_metadata( + app_name: str, app_version_details: dict, migrated: bool | None = None, custom_app: bool = False, +): migrated = get_app_metadata(app_name).get('migrated', False) if migrated is None else migrated with open(get_installed_app_metadata_path(app_name), 'w') as f: f.write(yaml.safe_dump({ 'metadata': app_version_details['app_metadata'], 'migrated': migrated, + 'custom_app': custom_app, **{k: app_version_details[k] for k in ('version', 'human_version')}, **get_portals_and_app_notes(app_name, app_version_details['version']), + # TODO: We should not try to get portals for custom apps for now })) diff --git a/src/middlewared/middlewared/plugins/apps/ix_apps/path.py b/src/middlewared/middlewared/plugins/apps/ix_apps/path.py index effd7003271d0..d0f0f73854cd0 100644 --- a/src/middlewared/middlewared/plugins/apps/ix_apps/path.py +++ b/src/middlewared/middlewared/plugins/apps/ix_apps/path.py @@ -47,5 +47,9 @@ def get_installed_app_config_path(app_name: str, version: str) -> str: return os.path.join(get_installed_app_version_path(app_name, version), 'user_config.yaml') +def get_installed_custom_app_compose_file(app_name: str, version: str) -> str: + return os.path.join(get_installed_app_rendered_dir_path(app_name, version), 'docker-compose.yaml') + + def get_installed_app_rendered_dir_path(app_name: str, version: str) -> str: return os.path.join(get_installed_app_version_path(app_name, version), 'templates/rendered') diff --git a/src/middlewared/middlewared/plugins/apps/ix_apps/portals.py b/src/middlewared/middlewared/plugins/apps/ix_apps/portals.py index db34de6b4a2dd..5a547ab9867b7 100644 --- a/src/middlewared/middlewared/plugins/apps/ix_apps/portals.py +++ b/src/middlewared/middlewared/plugins/apps/ix_apps/portals.py @@ -1,10 +1,6 @@ -import contextlib - -import yaml - from apps_validation.portals import IX_NOTES_KEY, IX_PORTAL_KEY, validate_portals_and_notes, ValidationErrors -from .lifecycle import get_rendered_templates_of_app +from .lifecycle import get_rendered_template_config_of_app def normalized_port_value(scheme: str, port: int) -> str: @@ -12,12 +8,7 @@ def normalized_port_value(scheme: str, port: int) -> str: def get_portals_and_app_notes(app_name: str, version: str) -> dict: - rendered_config = {} - for rendered_file in get_rendered_templates_of_app(app_name, version): - with contextlib.suppress(FileNotFoundError, yaml.YAMLError): - with open(rendered_file, 'r') as f: - rendered_config.update(yaml.safe_load(f.read())) - + rendered_config = get_rendered_template_config_of_app(app_name, version) portal_and_notes_config = { k: rendered_config[k] for k in (IX_NOTES_KEY, IX_PORTAL_KEY) diff --git a/src/middlewared/middlewared/plugins/apps/ix_apps/query.py b/src/middlewared/middlewared/plugins/apps/ix_apps/query.py index c20c78ff8a6d5..6930a4c4e514e 100644 --- a/src/middlewared/middlewared/plugins/apps/ix_apps/query.py +++ b/src/middlewared/middlewared/plugins/apps/ix_apps/query.py @@ -26,7 +26,11 @@ def __hash__(self): def upgrade_available_for_app( version_mapping: dict[str, dict[str, dict[str, str]]], app_metadata: dict ) -> bool: - if version_mapping.get(app_metadata['train'], {}).get(app_metadata['name']): + # TODO: Eventually we would want this to work as well but this will always require middleware changes + # depending on what new functionality we want introduced for custom app, so let's take care of this at that point + if (app_metadata['name'] == 'custom-app' and app_metadata['train'] == 'stable') is False and version_mapping.get( + app_metadata['train'], {} + ).get(app_metadata['name']): return parse_version(app_metadata['version']) < parse_version( version_mapping[app_metadata['train']][app_metadata['name']]['version'] ) diff --git a/src/middlewared/middlewared/plugins/apps/ix_apps/setup.py b/src/middlewared/middlewared/plugins/apps/ix_apps/setup.py index b595d06a752cb..b7f91981a9c76 100644 --- a/src/middlewared/middlewared/plugins/apps/ix_apps/setup.py +++ b/src/middlewared/middlewared/plugins/apps/ix_apps/setup.py @@ -1,14 +1,29 @@ import os import shutil +import textwrap +import yaml from .metadata import update_app_yaml_for_last_update from .path import get_app_parent_config_path, get_installed_app_version_path -def setup_install_app_dir(app_name: str, app_version_details: dict): +def setup_install_app_dir(app_name: str, app_version_details: dict, custom_app: bool = False): os.makedirs(os.path.join(get_app_parent_config_path(), app_name, 'versions'), exist_ok=True) to_install_app_version = os.path.basename(app_version_details['version']) destination = get_installed_app_version_path(app_name, to_install_app_version) - shutil.copytree(app_version_details['location'], destination) + if custom_app: + # TODO: See if it makes sense to creat a dummy app on apps side instead + os.makedirs(os.path.join(destination, 'templates/rendered'), exist_ok=True) + with open(os.path.join(destination, 'README.md'), 'w') as f: + f.write(textwrap.dedent(''' + # Custom App + + This is a custom app where user can use his/her own docker compose file for deploying services. + ''')) + + with open(os.path.join(destination, 'app.yaml'), 'w') as f: + f.write(yaml.safe_dump(app_version_details['app_metadata'])) + else: + shutil.copytree(app_version_details['location'], destination) update_app_yaml_for_last_update(destination, app_version_details['last_update']) diff --git a/src/middlewared/middlewared/plugins/catalog/apps.py b/src/middlewared/middlewared/plugins/catalog/apps.py index 3b3e8f875991b..4c236ab7166fe 100644 --- a/src/middlewared/middlewared/plugins/catalog/apps.py +++ b/src/middlewared/middlewared/plugins/catalog/apps.py @@ -17,7 +17,7 @@ async def latest(self, filters, options): return filter_list( await self.middleware.call( 'app.available', [ - ['last_update', '!=', None], ['name', '!=', 'ix-chart'], + ['last_update', '!=', None], ['name', '!=', 'ix-app'], ], {'order_by': ['-last_update']} ), filters, options ) @@ -58,8 +58,8 @@ def available(self, filters, options): results = [] installed_apps = [ - (app['chart_metadata']['name'], app['catalog'], app['catalog_train']) - for app in [] + (app['metadata']['name'], app['metadata']['train']) + for app in self.middleware.call_sync('app.query') ] catalog = self.middleware.call_sync('catalog.config') @@ -70,7 +70,7 @@ def available(self, filters, options): for app_data in train_data.values(): results.append({ 'catalog': catalog['label'], - 'installed': (app_data['name'], catalog['label'], train) in installed_apps, + 'installed': (app_data['name'], train) in installed_apps, 'train': train, **app_data, })