Messages: save blobs in the database instead of in the blob storage (#…

…368) * feat: add PersistMessageBodyInDb migration * feat: update the way of determining whether the message body is already filled * feat: remove the builder.Ignore(a => a.Body) statement * feat: update condition on whether the message body should be filled from blob storage * chore: add required project and package references * test: retrieve message body from blob storage if unavailable in database * test: retrieve message body from blob storage if unavailable in database * feat: remove saving message bodies to blob storage * chore: fix the formatting issues causing the ci/cd test to fail * chore: add project reference needed to use proper TestDataGenerator * fix: await messageRepository.FindMessagesWithIds() to prevent blocking statements * feat: make the Body column nullable and without a default value * chore: extract FillBody logic into a separate method * chore: fix indentation * fix: add missing semicolon * chore: refactor test as per requests in the PR * feat: make the Body column nullable and without a default value in SQL server migration as well * chore: fix the formatting issues causing the ci/cd test to fail * feat: update messages in the database from blob storage on project start if blob storage configuration is present * chore: remove obsolete test * fix: instance BlobStorageOptions to make it possible to run the project when blob storage settings are missing * chore: fix the formatting issues that caused the ci/cd tests fail * chore: move CreateRandomDeviceId() to building blocks for easier access in all modules * chore: add comment to refactor this class so that all necessary methods are contained only in BuildingBlocks.UnitTestTools * chore: remove reference to Devices.Application.Tests * chore: refactor code to match the changes made after requests in the Relationships PR * chore: remove duplicates of methods present in UnitTestTools * chore: update method calls to use unit test tools * chore: fix formatting issues * chore: fix missing indentation * chore: change var name * chore: fix formatting issues --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
nmshd · Nov 10, 2023 · 0d12e2b · 0d12e2b
1 parent eee9155
commit 0d12e2b
Show file tree

Hide file tree

Showing 34 changed files with 2,019 additions and 1,500 deletions.
diff --git a/BuildingBlocks/src/UnitTestTools/Data/TestDataGenerator.cs b/BuildingBlocks/src/UnitTestTools/Data/TestDataGenerator.cs
@@ -17,6 +17,11 @@ public static IdentityAddress CreateRandomIdentityAddress()
         return IdentityAddress.Create(CreateRandomBytes(), "id1");
     }
 
+    public static DeviceId CreateRandomDeviceId()
+    {
+        return DeviceId.New();
+    }
+
     public static byte[] CreateRandomBytes()
     {
         var random = new Random();

diff --git a/ConsumerApi/MessagesDbContextSeeder.cs b/ConsumerApi/MessagesDbContextSeeder.cs
@@ -0,0 +1,44 @@
+using Backbone.BuildingBlocks.API.Extensions;
+using Backbone.BuildingBlocks.Application.Abstractions.Infrastructure.Persistence.BlobStorage;
+using Backbone.Modules.Messages.Application.Infrastructure.Persistence;
+using Backbone.Modules.Messages.Infrastructure.Persistence.Database;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Options;
+
+namespace Backbone.ConsumerApi;
+
+public class MessagesDbContextSeeder : IDbSeeder<MessagesDbContext>
+{
+    private readonly IBlobStorage? _blobStorage;
+    private readonly string? _blobRootFolder;
+
+    public MessagesDbContextSeeder(IServiceProvider serviceProvider)
+    {
+        _blobStorage = serviceProvider.GetService<IBlobStorage>();
+        _blobRootFolder = serviceProvider.GetService<IOptions<BlobOptions>>()?.Value.RootFolder;
+    }
+
+    public async Task SeedAsync(MessagesDbContext context)
+    {
+        await FillBodyColumnsFromBlobStorage(context);
+    }
+
+    private async Task FillBodyColumnsFromBlobStorage(MessagesDbContext context)
+    {
+        // _blobRootFolder is null when blob storage configuration is not provided, meaning the content of database entries should not be loaded from blob storage
+        if (_blobRootFolder == null)
+            return;
+
+        // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
+        var messagesWithMissingBody = await context.Messages.Where(m => m.Body == null).ToListAsync();
+
+        foreach (var message in messagesWithMissingBody)
+        {
+            var blobMessageBody = await _blobStorage.FindAsync(_blobRootFolder, message.Id);
+            message.LoadBody(blobMessageBody);
+            context.Messages.Update(message);
+        }
+
+        await context.SaveChangesAsync();
+    }
+}
diff --git a/ConsumerApi/Program.cs b/ConsumerApi/Program.cs
@@ -111,7 +111,8 @@ static WebApplication CreateApp(string[] args)
 
     app
         .SeedDbContext<DevicesDbContext, DevicesDbContextSeeder>()
-        .SeedDbContext<QuotasDbContext, QuotasDbContextSeeder>();
+        .SeedDbContext<QuotasDbContext, QuotasDbContextSeeder>()
+        .SeedDbContext<MessagesDbContext, MessagesDbContextSeeder>();
 
     foreach (var module in app.Services.GetRequiredService<IEnumerable<AbstractModule>>())
     {
@@ -127,6 +128,7 @@ static void ConfigureServices(IServiceCollection services, IConfiguration config
 
     services.AddTransient<DevicesDbContextSeeder>();
     services.AddTransient<QuotasDbContextSeeder>();
+    services.AddTransient<MessagesDbContextSeeder>();
 
     services
         .AddModule<ChallengesModule>(configuration)

diff --git a/ConsumerApi/appsettings.override.json b/ConsumerApi/appsettings.override.json
@@ -75,11 +75,6 @@
                     "Provider": "Postgres",
                     "ConnectionString": "User ID=messages;Password=Passw0rd;Server=postgres;Port=5432;Database=enmeshed;" // postgres
                     // "ConnectionString": "Server=ms-sql-server;Database=enmeshed;User Id=messages;Password=Passw0rd;TrustServerCertificate=True" // sqlserver
-                },
-                "BlobStorage": {
-                    "CloudProvider": "Azure",
-                    "ConnectionInfo": "",
-                    "ContainerName": ""
                 }
             }
         },

diff --git a/Modules/Devices/test/Devices.Application.Tests/TestDataGenerator.cs b/Modules/Devices/test/Devices.Application.Tests/TestDataGenerator.cs
@@ -1,29 +1,11 @@
-using Backbone.DevelopmentKit.Identity.ValueObjects;
 using Backbone.Modules.Devices.Domain.Aggregates.Tier;
 using Backbone.Modules.Devices.Domain.Entities;
+using static Backbone.UnitTestTools.Data.TestDataGenerator;
 
 namespace Backbone.Modules.Devices.Application.Tests;
 
 public static class TestDataGenerator
 {
-    public static IdentityAddress CreateRandomIdentityAddress()
-    {
-        return IdentityAddress.Create(CreateRandomBytes(), "id1");
-    }
-
-    public static DeviceId CreateRandomDeviceId()
-    {
-        return DeviceId.New();
-    }
-
-    public static byte[] CreateRandomBytes()
-    {
-        var random = new Random();
-        var bytes = new byte[10];
-        random.NextBytes(bytes);
-        return bytes;
-    }
-
     public static TierId CreateRandomTierId()
     {
         return TierId.Generate();

diff --git a/...s/test/Devices.Application.Tests/Tests/Identities/Commands/UpdateIdentity/HandlerTests.cs b/...s/test/Devices.Application.Tests/Tests/Identities/Commands/UpdateIdentity/HandlerTests.cs
@@ -11,6 +11,7 @@
 using FakeItEasy;
 using FluentAssertions;
 using Xunit;
+using static Backbone.UnitTestTools.Data.TestDataGenerator;
 
 namespace Backbone.Modules.Devices.Application.Tests.Tests.Identities.Commands.UpdateIdentity;
 public class HandlerTests
@@ -25,7 +26,7 @@ public async void Updates_the_identity_in_the_database()
         var oldTier = new Tier(TierName.Create("Old tier").Value);
         var newTier = new Tier(TierName.Create("New Tier").Value);
 
-        var identity = new Identity(TestDataGenerator.CreateRandomDeviceId(), TestDataGenerator.CreateRandomIdentityAddress(), new byte[] { 1, 1, 1, 1, 1 }, oldTier.Id, 1);
+        var identity = new Identity(CreateRandomDeviceId(), CreateRandomIdentityAddress(), new byte[] { 1, 1, 1, 1, 1 }, oldTier.Id, 1);
 
         A.CallTo(() => identitiesRepository.FindByAddress(identity.Address, A<CancellationToken>._, A<bool>._)).Returns(identity);
         A.CallTo(() => tiersRepository.FindByIds(A<IEnumerable<TierId>>._, A<CancellationToken>._)).Returns(new List<Tier>() { oldTier, newTier });
@@ -54,7 +55,7 @@ public async void Publishes_TierOfIdentityChangedIntegrationEvent()
         var oldTier = new Tier(TierName.Create("Old tier").Value);
         var newTier = new Tier(TierName.Create("New Tier").Value);
 
-        var identity = new Identity(TestDataGenerator.CreateRandomDeviceId(), TestDataGenerator.CreateRandomIdentityAddress(), new byte[] { 1, 1, 1, 1, 1 }, oldTier.Id, 1);
+        var identity = new Identity(CreateRandomDeviceId(), CreateRandomIdentityAddress(), new byte[] { 1, 1, 1, 1, 1 }, oldTier.Id, 1);
 
         A.CallTo(() => identitiesRepository.FindByAddress(identity.Address, A<CancellationToken>._, A<bool>._)).Returns(identity);
         A.CallTo(() => tiersRepository.FindByIds(A<IEnumerable<TierId>>._, A<CancellationToken>._)).Returns(new List<Tier>() { oldTier, newTier });
@@ -79,7 +80,7 @@ public void Fails_when_identity_does_not_exist()
         var oldTier = new Tier(TierName.Create("Tier name").Value);
         var newTier = new Tier(TierName.Create("Tier name").Value);
 
-        var identity = new Identity(TestDataGenerator.CreateRandomDeviceId(), TestDataGenerator.CreateRandomIdentityAddress(), new byte[] { 1, 1, 1, 1, 1 }, oldTier.Id, 1);
+        var identity = new Identity(CreateRandomDeviceId(), CreateRandomIdentityAddress(), new byte[] { 1, 1, 1, 1, 1 }, oldTier.Id, 1);
 
         A.CallTo(() => tiersRepository.FindByIds(A<IEnumerable<TierId>>._, A<CancellationToken>._)).Returns(new List<Tier>() { oldTier, newTier });
         A.CallTo(() => identitiesRepository.FindByAddress(A<IdentityAddress>._, A<CancellationToken>._, A<bool>._)).Returns((Identity)null);
@@ -106,7 +107,7 @@ public void Fails_when_tier_does_not_exist()
         var oldTier = new Tier(TierName.Create("Tier name").Value);
         var newTier = new Tier(TierName.Create("Tier name").Value);
 
-        var identity = new Identity(TestDataGenerator.CreateRandomDeviceId(), TestDataGenerator.CreateRandomIdentityAddress(), new byte[] { 1, 1, 1, 1, 1 }, oldTier.Id, 1);
+        var identity = new Identity(CreateRandomDeviceId(), CreateRandomIdentityAddress(), new byte[] { 1, 1, 1, 1, 1 }, oldTier.Id, 1);
 
         A.CallTo(() => identitiesRepository.FindByAddress(identity.Address, A<CancellationToken>._, A<bool>._)).Returns(identity);
         A.CallTo(() => tiersRepository.FindByIds(A<IEnumerable<TierId>>._, A<CancellationToken>._)).Returns(new List<Tier>() { oldTier });
@@ -133,7 +134,7 @@ public void Does_nothing_when_tiers_are_the_same()
 
         var oldAndNewTier = new Tier(TierName.Create("Tier name").Value);
 
-        var identity = new Identity(TestDataGenerator.CreateRandomDeviceId(), TestDataGenerator.CreateRandomIdentityAddress(), new byte[] { 1, 1, 1, 1, 1 }, oldAndNewTier.Id, 1);
+        var identity = new Identity(CreateRandomDeviceId(), CreateRandomIdentityAddress(), new byte[] { 1, 1, 1, 1, 1 }, oldAndNewTier.Id, 1);
 
         A.CallTo(() => identitiesRepository.FindByAddress(identity.Address, A<CancellationToken>._, A<bool>._)).Returns(identity);
         A.CallTo(() => tiersRepository.FindByIds(A<IEnumerable<TierId>>._, A<CancellationToken>._)).Returns(new List<Tier> { oldAndNewTier });

diff --git a/...vices/test/Devices.Application.Tests/Tests/Identities/Queries/GetIdentity/HandlerTests.cs b/...vices/test/Devices.Application.Tests/Tests/Identities/Queries/GetIdentity/HandlerTests.cs
@@ -7,6 +7,7 @@
 using FakeItEasy;
 using FluentAssertions;
 using Xunit;
+using static Backbone.UnitTestTools.Data.TestDataGenerator;
 
 namespace Backbone.Modules.Devices.Application.Tests.Tests.Identities.Queries.GetIdentity;
 public class HandlerTests
@@ -15,7 +16,7 @@ public class HandlerTests
     public async void Gets_identity_by_address()
     {
         // Arrange
-        var identity = new Identity(TestDataGenerator.CreateRandomDeviceId(), TestDataGenerator.CreateRandomIdentityAddress(), new byte[] { 1, 1, 1, 1, 1 }, TestDataGenerator.CreateRandomTierId(), 1);
+        var identity = new Identity(CreateRandomDeviceId(), CreateRandomIdentityAddress(), new byte[] { 1, 1, 1, 1, 1 }, TestDataGenerator.CreateRandomTierId(), 1);
 
         var handler = CreateHandler(new FindByAddressStubRepository(identity));
 

diff --git a/...es/test/Devices.Application.Tests/Tests/Identities/Queries/ListIdentities/HandlerTests.cs b/...es/test/Devices.Application.Tests/Tests/Identities/Queries/ListIdentities/HandlerTests.cs
@@ -5,6 +5,7 @@
 using FluentAssertions;
 using FluentAssertions.Execution;
 using Xunit;
+using static Backbone.UnitTestTools.Data.TestDataGenerator;
 
 namespace Backbone.Modules.Devices.Application.Tests.Tests.Identities.Queries.ListIdentities;
 
@@ -37,14 +38,14 @@ public async void Returns_a_list_of_all_existing_identities()
         var request = new PaginationFilter();
         List<Identity> identitiesList = new()
         {
-            new(TestDataGenerator.CreateRandomDeviceId(),
-            TestDataGenerator.CreateRandomIdentityAddress(),
-            TestDataGenerator.CreateRandomBytes(),
+            new(CreateRandomDeviceId(),
+            CreateRandomIdentityAddress(),
+            CreateRandomBytes(),
             TestDataGenerator.CreateRandomTierId(),
             1),
-            new(TestDataGenerator.CreateRandomDeviceId(),
-            TestDataGenerator.CreateRandomIdentityAddress(),
-            TestDataGenerator.CreateRandomBytes(),
+            new(CreateRandomDeviceId(),
+            CreateRandomIdentityAddress(),
+            CreateRandomBytes(),
             TestDataGenerator.CreateRandomTierId(),
             1)
         };
@@ -63,8 +64,8 @@ public async void Returned_identities_have_all_properties_filled_as_expected()
     {
         // Arrange
         var request = new PaginationFilter();
-        var expectedClientId = TestDataGenerator.CreateRandomDeviceId();
-        var expectedAddress = TestDataGenerator.CreateRandomIdentityAddress();
+        var expectedClientId = CreateRandomDeviceId();
+        var expectedAddress = CreateRandomIdentityAddress();
         var expectedTierId = TestDataGenerator.CreateRandomTierId();
         List<Identity> identitiesList = new()
         {

diff --git a/Modules/Messages/src/Messages.ConsumerApi/Configuration.cs b/Modules/Messages/src/Messages.ConsumerApi/Configuration.cs
@@ -16,8 +16,7 @@ public class InfrastructureConfiguration
         [Required]
         public SqlDatabaseConfiguration SqlDatabase { get; set; } = new();
 
-        [Required]
-        public BlobStorageConfiguration BlobStorage { get; set; } = new();
+        public BlobStorageConfiguration? BlobStorage { get; set; }
 
         public class BlobStorageConfiguration
         {

diff --git a/Modules/Messages/src/Messages.ConsumerApi/MessagesModule.cs b/Modules/Messages/src/Messages.ConsumerApi/MessagesModule.cs
@@ -27,12 +27,17 @@ public override void ConfigureServices(IServiceCollection services, IConfigurati
             options.DbOptions.Provider = parsedConfiguration.Infrastructure.SqlDatabase.Provider;
             options.DbOptions.DbConnectionString = parsedConfiguration.Infrastructure.SqlDatabase.ConnectionString;
 
-            options.BlobStorageOptions.CloudProvider = parsedConfiguration.Infrastructure.BlobStorage.CloudProvider;
-            options.BlobStorageOptions.ConnectionInfo = parsedConfiguration.Infrastructure.BlobStorage.ConnectionInfo;
-            options.BlobStorageOptions.Container =
-                parsedConfiguration.Infrastructure.BlobStorage.ContainerName.IsNullOrEmpty()
-                    ? "messages"
-                    : parsedConfiguration.Infrastructure.BlobStorage.ContainerName;
+            if (parsedConfiguration.Infrastructure.BlobStorage != null)
+            {
+                options.BlobStorageOptions = new()
+                {
+                    CloudProvider = parsedConfiguration.Infrastructure.BlobStorage.CloudProvider,
+                    ConnectionInfo = parsedConfiguration.Infrastructure.BlobStorage.ConnectionInfo,
+                    Container = parsedConfiguration.Infrastructure.BlobStorage.ContainerName.IsNullOrEmpty()
+                        ? "messages"
+                        : parsedConfiguration.Infrastructure.BlobStorage.ContainerName
+                };
+            }
         });
 
         services.AddApplication();

diff --git a/Modules/Messages/src/Messages.Domain/Entities/Message.cs b/Modules/Messages/src/Messages.Domain/Entities/Message.cs
@@ -37,7 +37,7 @@ public Message(IdentityAddress createdBy, DeviceId createdByDevice, DateTime? do
 
     public void LoadBody(byte[] bytes)
     {
-        if (Body != null)
+        if (Body is { Length: > 0 })
         {
             throw new InvalidOperationException($"The Body of the message {Id} is already filled. It is not possible to change it.");
         }