Map missing chain parameter updates (#141)

* add missing chain update types to mapping

* Updated frontend

* updated chain log and bumped version

* fix data corruption

* refactor JobRepository

* factor to use common

* remove unused generic

* refactored health and added resilience to job

* added metric

* added test

* use default source class enricher from serilog

* Add job to migrate events

* add test

* add tests

* fix naming and add job

* update log format

* add log statement

* Update ImportWriteController.cs

* bump version

* Fix job name

backend/Application/Api/GraphQL/EfCore/Converters/Json/ChainUpdatePayloadConverter.cs

@@ -23,9 +23,14 @@ public class ChainUpdatePayloadConverter : PolymorphicJsonConverter<ChainUpdateP
         { typeof(PoolParametersChainUpdatePayload), 15 },
         { typeof(TimeParametersChainUpdatePayload), 16 },
         { typeof(MintDistributionV1ChainUpdatePayload), 17 },
+        { typeof(GasRewardsCpv2Update), 18 },
+        { typeof(BlockEnergyLimitUpdate), 19 },
+        { typeof(FinalizationCommitteeParametersUpdate), 20 },
+        { typeof(TimeoutParametersUpdate), 21 },
+        { typeof(MinBlockTimeUpdate), 22 },
     };
 
     public ChainUpdatePayloadConverter() : base(SerializeMap)
     {
     }
-}
+}

backend/Application/Api/GraphQL/Import/ImportWriteController.cs

@@ -11,7 +11,6 @@
 using Application.Configurations;
 using Application.Database;
 using Application.Database.MigrationJobs;
-using Application.Entities;
 using Application.Import;
 using Application.Jobs;
 using Application.Observability;

backend/Application/Api/GraphQL/Ratio.cs

@@ -0,0 +1,6 @@
+namespace Application.Api.GraphQL;
+
+public readonly record struct Ratio(ulong Numerator, ulong Denominator)
+{
+    internal static Ratio From(Concordium.Sdk.Types.Ratio ratio) => new(ratio.Numerator, ratio.Denominator);
+}

backend/Application/Api/GraphQL/Transactions/ChainUpdatePayload.cs

@@ -9,9 +9,8 @@ namespace Application.Api.GraphQL.Transactions;
 [UnionType]
 public abstract record ChainUpdatePayload
 {
-    internal static bool TryFrom(IUpdatePayload payload, out ChainUpdatePayload? chainUpdatePayload)
-    {
-        chainUpdatePayload = payload switch
+    internal static ChainUpdatePayload From(IUpdatePayload payload) =>
+        payload switch
         {
             AddAnonymityRevokerUpdate addAnonymityRevokerUpdate => AddAnonymityRevokerChainUpdatePayload.From(addAnonymityRevokerUpdate),
             AddIdentityProviderUpdate addIdentityProviderUpdate => AddIdentityProviderChainUpdatePayload.From(addIdentityProviderUpdate),
@@ -30,16 +29,67 @@ internal static bool TryFrom(IUpdatePayload payload, out ChainUpdatePayload? cha
             RootUpdate rootUpdate => RootKeysChainUpdatePayload.From(rootUpdate),
             TimeParametersCpv1Update timeParametersCpv1Update => TimeParametersChainUpdatePayload.From(timeParametersCpv1Update),
             TransactionFeeDistributionUpdate transactionFeeDistributionUpdate => TransactionFeeDistributionChainUpdatePayload.From(transactionFeeDistributionUpdate),
-            GasRewardsCpv2Update => null,
-            BlockEnergyLimitUpdate => null,
-            FinalizationCommitteeParametersUpdate => null,
-            TimeoutParametersUpdate => null,
-            MinBlockTimeUpdate => null,
+            Concordium.Sdk.Types.GasRewardsCpv2Update update => GasRewardsCpv2Update.From(update),
+            Concordium.Sdk.Types.BlockEnergyLimitUpdate update => BlockEnergyLimitUpdate.From(update),
+            Concordium.Sdk.Types.FinalizationCommitteeParametersUpdate update => FinalizationCommitteeParametersUpdate.From(update),
+            Concordium.Sdk.Types.TimeoutParametersUpdate update => TimeoutParametersUpdate.From(update),
+            Concordium.Sdk.Types.MinBlockTimeUpdate update => MinBlockTimeUpdate.From(update),
             _ => throw new ArgumentOutOfRangeException(nameof(payload))
         };
-
-        return chainUpdatePayload != null;
-    }
+}
+
+public sealed record MinBlockTimeUpdate(ulong DurationSeconds) : ChainUpdatePayload
+{
+    internal static MinBlockTimeUpdate From(Concordium.Sdk.Types.MinBlockTimeUpdate update) =>
+        new((ulong)update.Duration.TotalSeconds);
+
+}
+
+public sealed record TimeoutParametersUpdate(
+    ulong DurationSeconds, Ratio Increase, Ratio Decrease
+    ) : ChainUpdatePayload
+{
+    internal static TimeoutParametersUpdate From(Concordium.Sdk.Types.TimeoutParametersUpdate update) =>
+        new(
+            (ulong)update.TimeoutParameters.Duration.TotalSeconds,
+            Ratio.From(update.TimeoutParameters.Increase),
+            Ratio.From(update.TimeoutParameters.Decrease)
+        );
+}
+
+public sealed record FinalizationCommitteeParametersUpdate(
+    uint MinFinalizers,
+    uint MaxFinalizers,
+    decimal FinalizersRelativeStakeThreshold
+    ) : ChainUpdatePayload
+{
+    internal static FinalizationCommitteeParametersUpdate From(
+        Concordium.Sdk.Types.FinalizationCommitteeParametersUpdate update) =>
+        new FinalizationCommitteeParametersUpdate(
+            update.FinalizationCommitteeParameters.MinFinalizers,
+            update.FinalizationCommitteeParameters.MaxFinalizers,
+            update.FinalizationCommitteeParameters.FinalizersRelativeStakeThreshold.AsDecimal()
+        );
+}
+
+public sealed record BlockEnergyLimitUpdate(
+    ulong EnergyLimit) : ChainUpdatePayload
+{
+    internal static BlockEnergyLimitUpdate From(Concordium.Sdk.Types.BlockEnergyLimitUpdate update) => 
+        new(update.EnergyLimit.Value);
+}
+
+public sealed record GasRewardsCpv2Update(
+    decimal Baker,
+    decimal AccountCreation,
+    decimal ChainUpdate) : ChainUpdatePayload
+{
+    internal static GasRewardsCpv2Update From(Concordium.Sdk.Types.GasRewardsCpv2Update update) =>
+        new(
+            update.Baker.AsDecimal(),
+            update.AccountCreation.AsDecimal(),
+            update.ChainUpdate.AsDecimal()
+        );
 }
 
 public record ProtocolChainUpdatePayload(
@@ -274,4 +324,4 @@ internal static MintDistributionV1ChainUpdatePayload From(MintDistributionCpv1Up
         );
     }
 }
-
+

backend/Application/Api/GraphQL/Transactions/TransactionResultEvent.cs

@@ -125,10 +125,7 @@ internal static IEnumerable<TransactionResultEvent> ToIter(IBlockItemSummaryDeta
 
                 break;
             case UpdateDetails updateDetails:
-                if (ChainUpdateEnqueued.TryFrom(updateDetails, blockSlotTime, out var chainUpdateEnqueued))
-                {
-                    yield return chainUpdateEnqueued!;
-                }
+                yield return ChainUpdateEnqueued.From(updateDetails, blockSlotTime);
                 break;
         }
     }
@@ -878,20 +875,14 @@ public record ChainUpdateEnqueued(
     bool EffectiveImmediately,
     ChainUpdatePayload Payload) : TransactionResultEvent
 {
-    internal static bool TryFrom(
+    internal static ChainUpdateEnqueued From(
         UpdateDetails updateDetails,
-        DateTimeOffset blockSlotTime,
-        out ChainUpdateEnqueued? chainUpdateEnqueued)
+        DateTimeOffset blockSlotTime)
     {
-        if (!ChainUpdatePayload.TryFrom(updateDetails.Payload, out var chainUpdatePayload))
-        {
-            chainUpdateEnqueued = null;
-            return false;
-        } 
+        var chainUpdatePayload = ChainUpdatePayload.From(updateDetails.Payload);
         var isEffectiveImmediately = updateDetails.EffectiveTime.ToUnixTimeSeconds() == 0;
         var effectiveTime = isEffectiveImmediately ? blockSlotTime : updateDetails.EffectiveTime;
-        chainUpdateEnqueued = new ChainUpdateEnqueued(effectiveTime, isEffectiveImmediately, chainUpdatePayload!);
-        return true;
+        return new ChainUpdateEnqueued(effectiveTime, isEffectiveImmediately, chainUpdatePayload!);
     }
 }
 

backend/Application/Database/MigrationJobs/_00_UpdateValidatorCommissionRates.cs

@@ -23,7 +23,6 @@ public class _00_UpdateValidatorCommissionRates : IMainMigrationJob {
     /// WARNING - Do not change this if job already executed on environment, since it will trigger rerun of job.
     /// </summary>
     private const string JobName = "_00_UpdateValidatorCommissionRates";
-
     private readonly IDbContextFactory<GraphQlDbContext> _contextFactory;
     private readonly IConcordiumNodeClient _client;
     private readonly JobHealthCheck _jobHealthCheck;

backend/Application/Database/MigrationJobs/_01_AddMissingChainUpdateEvents.cs

@@ -0,0 +1,141 @@
+using System.Threading;
+using System.Threading.Tasks;
+using Application.Api.GraphQL.EfCore;
+using Application.Api.GraphQL.Import;
+using Application.Api.GraphQL.Transactions;
+using Application.Exceptions;
+using Application.Import.ConcordiumNode;
+using Application.Observability;
+using Application.Resilience;
+using Concordium.Sdk.Types;
+using Dapper;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Options;
+using Serilog.Context;
+
+namespace Application.Database.MigrationJobs;
+
+/// <summary>
+/// Some transaction events hasn't been mapped to the database. Those missing are
+/// <see cref="Concordium.Sdk.Types.UpdateType"/> of one of below values
+/// - <see cref="Concordium.Sdk.Types.UpdateType.GasRewardsCpv2Update"/>
+/// - <see cref="Concordium.Sdk.Types.UpdateType.TimeoutParametersUpdate"/>
+/// - <see cref="Concordium.Sdk.Types.UpdateType.MinBlockTimeUpdate"/>
+/// - <see cref="Concordium.Sdk.Types.UpdateType.BlockEnergyLimitUpdate"/>
+/// - <see cref="Concordium.Sdk.Types.UpdateType.FinalizationCommitteeParametersUpdate"/>
+///
+/// The events are mapped to <see cref="ChainUpdateEnqueued"/>. This migration job adds missing events.
+///
+/// Even though transaction events for these cases hasn't been mapped the event type has been set on the
+/// <see cref="Transaction"/> entity on property <see cref="Transaction.TransactionType"/>. The jobs starts by
+/// querying the transaction table for those transaction with is of one of the missing event types. 
+/// 
+/// The mapping between the <see cref="Transaction.TransactionType"/> and the string value stored in the database
+/// is present at <see cref="Application.Api.GraphQL.EfCore.Converters.EfCore.TransactionTypeToStringConverter"/>.
+///  
+/// There is an one-to-one relation between transaction and events when the transaction is of type
+/// <see cref="Concordium.Sdk.Types.UpdateDetails"/>. Hence the job is idempotent because it checks if any transaction
+/// event for the given transaction already exist, and only generates an event if none is present.
+/// </summary>
+public class _01_AddMissingChainUpdateEvents : IMainMigrationJob {
+    /// <summary>
+    /// WARNING - Do not change this if job already executed on environment, since it will trigger rerun of job.
+    /// </summary>
+    private const string JobName = "_01_AddMissingChainUpdateEvents";
+
+    /// <summary>
+    /// The mapping between the <see cref="Transaction.TransactionType"/> and the string value stored in the database
+    /// is present at <see cref="Application.Api.GraphQL.EfCore.Converters.EfCore.TransactionTypeToStringConverter"/>.
+    /// </summary>
+    private const string AffectedTransactionTypesSql = @"
+SELECT id as Id, block_id as BlockId, index as TransactionIndex, transaction_hash as TransactionHash
+FROM graphql_transactions
+WHERE transaction_type IN ('2.22', '2.21', '2.20', '2.19', '2.18');
+";    
+
+    private readonly IDbContextFactory<GraphQlDbContext> _contextFactory;
+    private readonly IConcordiumNodeClient _client;
+    private readonly JobHealthCheck _jobHealthCheck;
+    private readonly ILogger _logger;
+    private readonly MainMigrationJobOptions _mainMigrationJobOptions;
+
+    public _01_AddMissingChainUpdateEvents(
+        IDbContextFactory<GraphQlDbContext> contextFactory,
+        IConcordiumNodeClient client,
+        JobHealthCheck jobHealthCheck,
+        IOptions<MainMigrationJobOptions> options
+    )
+    {
+        _contextFactory = contextFactory;
+        _client = client;
+        _jobHealthCheck = jobHealthCheck;
+        _logger = Log.ForContext<_00_UpdateValidatorCommissionRates>();
+        _mainMigrationJobOptions = options.Value;
+    }
+
+    /// <summary>
+    /// Start import of missing transaction events.
+    /// </summary>
+    /// <exception cref="JobException">If the transaction fetched from the node isn't
+    /// <see cref="TransactionStatusFinalized"/> or the transaction isn't of type <see cref="UpdateDetails"/>
+    /// </exception>
+    public async Task StartImport(CancellationToken token)
+    {
+        using var _ = TraceContext.StartActivity(GetUniqueIdentifier());
+        using var __ = LogContext.PushProperty("Job", GetUniqueIdentifier());
+
+        try
+        {
+            await Policies.GetTransientPolicy(GetUniqueIdentifier(), _logger, _mainMigrationJobOptions.RetryCount, _mainMigrationJobOptions.RetryDelay)
+                .ExecuteAsync(async () =>
+                {
+                    await using var context = await _contextFactory.CreateDbContextAsync(token);
+                    var connection = context.Database.GetDbConnection();
+
+                    var transactions = await connection.QueryAsync<Transaction>(AffectedTransactionTypesSql);
+
+                    foreach (var transaction in transactions)
+                    {
+                        var count = await context.TransactionResultEvents
+                            .Where(te => te.TransactionId == transaction.Id)
+                            .CountAsync(cancellationToken: token);
+                        // If a transaction event exist the event has already been generated. 
+                        if (count > 0)
+                        {
+                            continue;
+                        }
+
+                        var blockItemStatus = await _client.GetBlockItemStatusAsync(TransactionHash.From(transaction.TransactionHash), token);
+
+                        var finalized = blockItemStatus.GetFinalizedBlockItemSummary();
+
+                        if (finalized.Details is not UpdateDetails updateDetails)
+                        {
+                            throw JobException.Create(GetUniqueIdentifier(),
+                                $"Transaction details was of wrong type {finalized.Details.GetType()}");
+                        }
+
+                        var block = await context
+                            .Blocks
+                            .SingleAsync(b => b.Id == transaction.BlockId, cancellationToken: token);
+
+                        var chainUpdateEnqueued = ChainUpdateEnqueued.From(updateDetails, block.BlockSlotTime);
+
+                        var transactionRelated = new TransactionRelated<TransactionResultEvent>(transaction.Id, 0, chainUpdateEnqueued);
+                        await context.TransactionResultEvents.AddAsync(transactionRelated, token);
+                        await context.SaveChangesAsync(token);
+                    }
+                });
+        }
+        catch (Exception e)
+        {
+            _jobHealthCheck.AddUnhealthyJobWithMessage(GetUniqueIdentifier(), "Job stopped due to exception.");
+            _logger.Fatal(e, $"{GetUniqueIdentifier()} stopped due to exception.");
+            throw;
+        }
+    }
+
+    public string GetUniqueIdentifier() => JobName;
+
+    public bool ShouldNodeImportAwait() => false;
+}

backend/Application/Exceptions/ConcordiumClientWrapperException.cs

@@ -0,0 +1,7 @@
+namespace Application.Exceptions;
+
+public sealed class ConcordiumClientWrapperException : Exception
+{
+    public ConcordiumClientWrapperException(string message) : base(message)
+    {}
+}

backend/Application/Exceptions/JobException.cs

@@ -0,0 +1,12 @@
+namespace Application.Exceptions;
+
+internal sealed class JobException : Exception
+{
+    private JobException(string message) : base(message)
+    {}
+
+    internal static JobException Create(string identifier, string message)
+    {
+        return new JobException($"Job {identifier} encountered error: {message}");
+    }
+}

backend/Application/Extensions/StartupExtensions.cs

@@ -31,7 +31,7 @@ internal static void AddMainMigrationJobs(this IServiceCollection collection, IC
         collection.AddSingleton<IJobRepository<MainMigrationJob>, JobRepository<MainMigrationJob>>();
 
         collection.AddTransient<IMainMigrationJob, _00_UpdateValidatorCommissionRates>();
-        collection.AddSingleton<IConcordiumNodeClient, ConcordiumNodeClient>();
+        collection.AddTransient<IMainMigrationJob, _01_AddMissingChainUpdateEvents>();
     }
 
     internal static void AddConcordiumClient(this IServiceCollection services, IConfiguration configuration)
@@ -40,6 +40,7 @@ internal static void AddConcordiumClient(this IServiceCollection services, IConf
         var concordiumClientOptions = configuration.GetSection("ConcordiumNodeGrpc").Get<ConcordiumClientOptions>();
         var uri = new Uri(grpcNodeClientSettings.Address);
         services.AddSingleton(new ConcordiumClient(uri, concordiumClientOptions));
+        services.AddSingleton<IConcordiumNodeClient, ConcordiumNodeClient>();
     }
 
     internal static void AddDefaultHealthChecks(this IServiceCollection services)

backend/Application/Import/ConcordiumNode/ConcordiumClientWrappers/BlockItemSummaryWrapper.cs

@@ -0,0 +1,37 @@
+using Application.Exceptions;
+using Concordium.Sdk.Types;
+
+namespace Application.Import.ConcordiumNode.ConcordiumClientWrappers;
+
+public interface IBlockItemSummaryWrapper
+{
+    ITransactionStatus GetTransactionStatus();
+    BlockItemSummary GetFinalizedBlockItemSummary();
+}
+
+public class BlockItemSummaryWrapper : IBlockItemSummaryWrapper
+{
+    private readonly ITransactionStatus _transactionStatus;
+
+    public BlockItemSummaryWrapper(ITransactionStatus transactionStatus)
+    {
+        _transactionStatus = transactionStatus;
+    }
+    public ITransactionStatus GetTransactionStatus() => _transactionStatus;
+
+    /// <summary>
+    /// Get block item summary for finalized transaction.
+    /// </summary>
+    /// <exception cref="ConcordiumClientWrapperException">Throws exception if the transaction
+    /// it not finalized.
+    /// </exception>
+    public BlockItemSummary GetFinalizedBlockItemSummary()
+    {
+        if (_transactionStatus is not TransactionStatusFinalized finalized)
+        {
+            throw new ConcordiumClientWrapperException($"Transaction was of wrong type {_transactionStatus.GetType()}");
+        }
+
+        return finalized.State.Summary;
+    }
+}