Introduce support for app config migrations

src/middlewared/middlewared/plugins/apps/migration_utils.py

@@ -0,0 +1,83 @@
+import os
+import yaml
+
+from apps_validation.json_schema_utils import APP_CONFIG_MIGRATIONS_SCHEMA
+from jsonschema import validate, ValidationError
+from pkg_resources import parse_version
+
+from middlewared.plugins.apps.ix_apps.path import get_installed_app_version_path
+
+
+def version_in_range(version: str, min_version: str = None, max_version: str = None) -> bool:
+    parsed_version = parse_version(version)
+    if min_version and max_version and (min_version == max_version):
+        return parsed_version == parse_version(min_version)
+    if min_version and parsed_version < parse_version(min_version):
+        return False
+    if max_version and parsed_version > parse_version(max_version):
+        return False
+    return True
+
+
+def validate_versions(current_version: str, target_version: str):
+    if not current_version or not target_version:
+        return {
+            'error': 'Both current and target version should be specified',
+            'migration_files': []
+        }
+    try:
+        if parse_version(current_version) >= parse_version(target_version):
+            return {
+                'error': 'Target version should be greater than current version',
+                'migration_files': []
+            }
+    except ValueError:
+        return {
+            'error': 'Both versions should be numeric string',
+            'migration_files': []
+        }
+
+    # If no error, return the default response
+    return {'error': None, 'migration_files': []}
+
+
+def get_migration_scripts(app_name: str, current_version: str, target_version: str) -> dict:
+    migration_files = validate_versions(current_version, target_version)
+    if migration_files['error']:
+        return migration_files
+
+    target_version_path = get_installed_app_version_path(app_name, target_version)
+    migration_yaml_path = os.path.join(target_version_path, 'app_migrations.yaml')
+
+    try:
+        with open(migration_yaml_path, 'r') as f:
+            data = yaml.safe_load(f)
+
+        validate(data, APP_CONFIG_MIGRATIONS_SCHEMA)
+    except FileNotFoundError:
+        return migration_files
+    except yaml.YAMLError:
+        migration_files['error'] = 'Invalid YAML'
+        return migration_files
+    except ValidationError:
+        migration_files['error'] = 'Data structure in the YAML file does not conform to the JSON schema'
+        return migration_files
+    else:
+        for migrations in data['migrations']:
+            migration_file = migrations['file']
+            from_constraint = migrations.get('from', {})
+            target_constraint = migrations.get('target', {})
+            if (
+                version_in_range(current_version, **from_constraint) and
+                version_in_range(target_version, **target_constraint)
+            ):
+                migration_file_path = os.path.join(target_version_path, f'migrations/{migration_file}')
+                if os.access(migration_file_path, os.X_OK):
+                    migration_files['migration_files'].append({'error': None, 'migration_file': migration_file_path})
+                else:
+                    migration_files['migration_files'].append({
+                        'error': f'{migration_file!r} Migration file is not executable',
+                        'migration_file': migration_file_path
+                    })
+
+        return migration_files

src/middlewared/middlewared/plugins/apps/upgrade.py

@@ -1,4 +1,9 @@
 import logging
+import os
+import subprocess
+import tempfile
+import yaml
+
 from pkg_resources import parse_version
 
 from middlewared.api import api_method
@@ -11,6 +16,7 @@
 from .ix_apps.lifecycle import add_context_to_values, get_current_app_config, update_app_config
 from .ix_apps.path import get_installed_app_path
 from .ix_apps.upgrade import upgrade_config
+from .migration_utils import get_migration_scripts
 from .version_utils import get_latest_version_from_app_versions
 from .utils import get_upgrade_snap_name
 
@@ -112,7 +118,7 @@ def upgrade(self, job, app_name, options):
         # 6) Update collective metadata config to reflect new version
         # 7) Finally create ix-volumes snapshot for rollback
         with upgrade_config(app_name, upgrade_version):
-            config = get_current_app_config(app_name, app['version'])
+            config = self.upgrade_values(app, upgrade_version)
             config.update(options['values'])
             new_values = self.middleware.call_sync(
                 'app.schema.normalize_and_validate_values', upgrade_version, config, False,
@@ -229,3 +235,53 @@ async def check_upgrade_alerts(self):
                 await self.middleware.call('alert.oneshot_create', 'AppUpdate', {'name': app['id']})
             else:
                 await self.middleware.call('alert.oneshot_delete', 'AppUpdate', app['id'])
+
+    @private
+    def get_data_for_upgrade_values(self, app, upgrade_version):
+        current_version = app['version']
+        target_version = upgrade_version['version']
+        migration_files_path = get_migration_scripts(app['name'], current_version, target_version)
+        config = get_current_app_config(app['name'], current_version)
+        file_paths = []
+
+        if migration_files_path['error']:
+            raise CallError(f'Failed to apply migrations: {migration_files_path["error"]}')
+        else:
+            errors = []
+            for migration_file in migration_files_path['migration_files']:
+                if migration_file['error']:
+                    errors.append(migration_file['error'])
+                else:
+                    file_paths.append(migration_file['migration_file'])
+
+            if errors:
+                errors_str = '\n'.join(errors)
+                raise CallError(f'Failed to upgrade because of following migration file(s) error(s):\n{errors_str}')
+
+        return file_paths, config
+
+    @private
+    def upgrade_values(self, app, upgrade_version):
+        migration_file_paths, config = self.get_data_for_upgrade_values(app, upgrade_version)
+        for migration_file_path in migration_file_paths:
+            with tempfile.NamedTemporaryFile(mode='w+') as f:
+                try:
+                    yaml.safe_dump(config, f, default_flow_style=False)
+                except yaml.YAMLError as e:
+                    raise CallError(f'Failed to dump config for {app["name"]}: {e}')
+
+                f.flush()
+                cp = subprocess.Popen([migration_file_path, f.name], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                stdout, stderr = cp.communicate()
+
+            migration_file_basename = os.path.basename(migration_file_path)
+            if cp.returncode:
+                raise CallError(f'Failed to execute {migration_file_basename!r} migration: {stderr.decode()}')
+
+            if stdout:
+                try:
+                    config = yaml.safe_load(stdout.decode())
+                except yaml.YAMLError as e:
+                    raise CallError(f'{migration_file_basename!r} migration file returned invalid YAML: {e}')
+
+        return config