Add support for strongly typed settings + inheritance

nexus-main · Feb 3, 2025 · 5176749 · 5176749
1 parent 354c8f2
commit 5176749
Show file tree

Hide file tree

Showing 9 changed files with 72 additions and 41 deletions.
diff --git a/src/Nexus/Extensibility/DataSource/DataSourceController.cs b/src/Nexus/Extensibility/DataSource/DataSourceController.cs
@@ -1066,15 +1066,12 @@ private Task SetContextAsync<T>(
         CancellationToken cancellationToken
     )
     {
-        var sourceConfiguration = registration.Configuration.ValueKind == JsonValueKind.Undefined
-            ? default
-            : JsonSerializer
-                .Deserialize<T>(registration.Configuration);
+        var sourceConfiguration = JsonSerializer.Deserialize<T>(registration.Configuration);
 
         var context = new DataSourceContext<T?>(
             ResourceLocator: registration.ResourceLocator,
-            RequestConfiguration: _requestConfiguration,
-            SourceConfiguration: sourceConfiguration
+            SourceConfiguration: sourceConfiguration,
+            RequestConfiguration: _requestConfiguration
         );
 
         return dataSource.SetContextAsync(context, logger, cancellationToken);

diff --git a/src/Nexus/Services/UpgradeConfigurationService.cs b/src/Nexus/Services/UpgradeConfigurationService.cs
@@ -47,8 +47,15 @@ public async Task UpgradeAsync()
 
                     try
                     {
-                        /* Find generic parameters */
                         var sourceType = _extensionHive.GetExtensionType(sourceTypeName);
+
+                        /* Upgrade */
+                        var upgradedConfiguration = await InternalUpgradeAllAsync(
+                            sourceType,
+                            registration.Configuration
+                        );
+
+                        /* Ensure deserialization works */
                         var sourceInterfaceTypes = sourceType.GetInterfaces();
 
                         if (!sourceInterfaceTypes.Contains(typeof(IUpgradableDataSource)))
@@ -64,22 +71,8 @@ public async Task UpgradeAsync()
                             throw new Exception("Data sources must implement IDataSource<T>.");
 
                         var configurationType = genericInterface.GenericTypeArguments[0];
-                        var timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(1));
-
-                        /* Invoke InternalUpgradeAsync */
-                        var methodInfo = typeof(UpgradeConfigurationService)
-                            .GetMethod(nameof(InternalUpgradeAsync), BindingFlags.NonPublic | BindingFlags.Static)!;
-
-                        var genericMethod = methodInfo
-                            .MakeGenericMethod(sourceType, configurationType);
 
-                        var upgradedConfiguration = await (Task<JsonElement>)genericMethod.Invoke(
-                            default,
-                            [
-                                registration.Configuration,
-                                timeoutTokenSource.Token
-                            ]
-                        )!;
+                        _ = JsonSerializer.Deserialize(upgradedConfiguration, configurationType, JsonSerializerOptions.Web);
 
                         /* Update pipeline */
                         if (!JsonElement.DeepEquals(registration.Configuration, upgradedConfiguration))
@@ -110,15 +103,35 @@ public async Task UpgradeAsync()
         }
     }
 
-    private static async Task<JsonElement> InternalUpgradeAsync<TSource, TConfiguration>(
-        JsonElement configuration,
-        CancellationToken cancellationToken
-    ) where TSource : IUpgradableDataSource
+    private static async Task<JsonElement> InternalUpgradeAllAsync(
+        Type sourceType,
+        JsonElement configuration
+    )
     {
-        var upgradedConfiguration = await TSource.UpgradeSourceConfigurationAsync(configuration, cancellationToken);
+        var upgradedConfiguration = configuration;
 
-        /* ensure it can be deserialized */
-        _ = JsonSerializer.Deserialize<TConfiguration>(upgradedConfiguration, JsonSerializerOptions.Web);
+        /* For each type in the inheritance chain */
+        var nextType = sourceType;
+
+        while (!(nextType is null || nextType == typeof(object)))
+        {
+            var currentType = nextType;
+            nextType = nextType.BaseType;
+
+            var timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(1));
+
+            /* Invoke InternalUpgradeAsync */
+            var methodInfo = currentType
+                .GetMethod(nameof(IUpgradableDataSource.UpgradeSourceConfigurationAsync), BindingFlags.Public | BindingFlags.Static)!;
+
+            upgradedConfiguration = await (Task<JsonElement>)methodInfo.Invoke(
+                default,
+                [
+                    upgradedConfiguration,
+                    timeoutTokenSource.Token
+                ]
+            )!;
+        }
 
         return upgradedConfiguration;
     }

diff --git a/src/clients/dotnet/NexusClient.g.cs b/src/clients/dotnet/NexusClient.g.cs
@@ -3004,7 +3004,7 @@ public record CatalogInfo(string Id, string? Title, string? Contact, string? Rea
     public record PipelineInfo(Guid Id, IReadOnlyList<string> Types, IReadOnlyList<string?> InfoUrls);
 
     /// <summary>
-    /// A data source time range.
+    /// A catalog time range.
     /// </summary>
     /// <param name="Begin">The date/time of the first data in the catalog.</param>
     /// <param name="End">The date/time of the last data in the catalog.</param>

diff --git a/src/clients/python/nexus_api/V1.py b/src/clients/python/nexus_api/V1.py
@@ -1907,7 +1907,7 @@ class PipelineInfo:
 @dataclass(frozen=True)
 class CatalogTimeRange:
     """
-    A data source time range.
+    A catalog time range.
 
     Args:
         begin: The date/time of the first data in the catalog.

diff --git a/src/extensibility/dotnet-extensibility/Extensibility/DataSource/DataSourceTypes.cs b/src/extensibility/dotnet-extensibility/Extensibility/DataSource/DataSourceTypes.cs
@@ -20,7 +20,7 @@ public record DataSourceContext<T>(
 );
 
 /// <summary>
-/// A data source time range.
+/// A catalog time range.
 /// </summary>
 /// <param name="Begin">The date/time of the first data in the catalog.</param>
 /// <param name="End">The date/time of the last data in the catalog.</param>

diff --git a/src/extensibility/dotnet-extensibility/Extensibility/DataSource/SimpleDataSource.cs b/src/extensibility/dotnet-extensibility/Extensibility/DataSource/SimpleDataSource.cs
@@ -25,7 +25,8 @@ public abstract class SimpleDataSource<T> : IDataSource<T>
     public Task SetContextAsync(
         DataSourceContext<T> context,
         ILogger logger,
-        CancellationToken cancellationToken)
+        CancellationToken cancellationToken
+    )
     {
         Context = context;
         Logger = logger;
@@ -36,17 +37,20 @@ public Task SetContextAsync(
     /// <inheritdoc />
     public abstract Task<CatalogRegistration[]> GetCatalogRegistrationsAsync(
         string path,
-        CancellationToken cancellationToken);
+        CancellationToken cancellationToken
+    );
 
     /// <inheritdoc />
     public abstract Task<ResourceCatalog> EnrichCatalogAsync(
         ResourceCatalog catalog,
-        CancellationToken cancellationToken);
+        CancellationToken cancellationToken
+    );
 
     /// <inheritdoc />
     public virtual Task<CatalogTimeRange> GetTimeRangeAsync(
         string catalogId,
-        CancellationToken cancellationToken)
+        CancellationToken cancellationToken
+    )
     {
         return Task.FromResult(new CatalogTimeRange(DateTime.MinValue, DateTime.MaxValue));
     }
@@ -68,5 +72,6 @@ public abstract Task ReadAsync(
         ReadRequest[] requests,
         ReadDataHandler readData,
         IProgress<double> progress,
-        CancellationToken cancellationToken);
+        CancellationToken cancellationToken
+    );
 }
diff --git a/src/extensibility/python-extensibility/nexus_extensibility/_extensibility_data_source.py b/src/extensibility/python-extensibility/nexus_extensibility/_extensibility_data_source.py
@@ -63,7 +63,7 @@ class DataSourceContext(Generic[T]):
 @dataclass(frozen=True)
 class CatalogTimeRange:
     """
-    A data source time range.
+    A catalog time range.
 
     Args:
         begin: The date/time of the first data in the catalog.
@@ -119,6 +119,7 @@ def __call__(self, resource_path: str, begin: datetime, end: datetime) -> Awaita
 
 ################# DATA SOURCE ###############
 
+# use this syntax in future (3.12+): IDataSource[T](ABC)
 class IDataSource(Generic[T], ABC):
     """
     A data source.

diff --git a/tests/Nexus.Tests/DataSource/TestSource.cs b/tests/Nexus.Tests/DataSource/TestSource.cs
@@ -14,15 +14,29 @@ public record TestSourceSettings(
     double Bar
 );
 
+public class TestSourceBase : IUpgradableDataSource
+{
+    public static Task<JsonElement> UpgradeSourceConfigurationAsync(JsonElement configuration, CancellationToken cancellationToken)
+    {
+        var configurationNode = (JsonSerializer.SerializeToNode(configuration) as JsonObject)!;
+
+        configurationNode["baz"] = configurationNode["version"]!.GetValue<int>();
+
+        var upgradedConfiguration = JsonSerializer.SerializeToElement(configurationNode);
+
+        return Task.FromResult(upgradedConfiguration);
+    }
+}
+
 [ExtensionDescription(
     "Augments existing catalogs with more awesome data.",
     "https://github.com/nexus-main/nexus",
     "https://github.com/nexus-main/nexus/blob/master/tests/Nexus.Tests/DataSource/TestSource.cs")]
-public class TestSource : IDataSource<TestSourceSettings?>, IUpgradableDataSource
+public class TestSource : TestSourceBase, IDataSource<TestSourceSettings?>, IUpgradableDataSource
 {
     public const string LocalCatalogId = "/SAMPLE/LOCAL";
 
-    public static Task<JsonElement> UpgradeSourceConfigurationAsync(
+    public static new Task<JsonElement> UpgradeSourceConfigurationAsync(
         JsonElement configuration,
         CancellationToken cancellationToken
     )

diff --git a/tests/Nexus.Tests/Services/UpgradeConfigurationServiceTests.cs b/tests/Nexus.Tests/Services/UpgradeConfigurationServiceTests.cs
@@ -44,7 +44,8 @@ registration with {
                         $$"""
                         {
                             "version": 2,
-                            "bar": 1.99
+                            "bar": 1.99,
+                            "baz": 2
                         }
                         """
                     ),