feat: cleanup old model revision files

Cognite.Simulator.Tests/SeedData.cs

@@ -23,6 +23,7 @@ public class SeedData
         public static string TestExtPipelineId = "utils-tests-pipeline-" + Now;
         public static string TestIntegrationExternalId = "utils-integration-tests-connector-" + Now;
         public static string TestModelExternalId = "Utils-Connector_Test_Model_" + Now;
+        public static string TestModelExternalId2 = "Utils-Connector_Test_Model_2_" + Now;
         public static string TestRoutineExternalId = "Test Routine with extended IO " + Now;
         public static string TestScheduledRoutineExternalId = "Test Scheduled Routine " + Now;
         public static string TestRoutineExternalIdWithTs = "Test Routine with Input TS and extended IO " + Now;
@@ -119,6 +120,12 @@ public static async Task<SimulatorModel> GetOrCreateSimulatorModel(Client sdk, S
             DataSetId = TestDataSetId,
         };
 
+        public static FileCreate SimpleModelFileCreate3 = new FileCreate() {
+            Name = "simutils-tests-model-3.out",
+            ExternalId = "simutils-tests-model-single-byte-3",
+            DataSetId = TestDataSetId,
+        };
+
         public static async Task<CogniteSdk.File> GetOrCreateFile(Client sdk, FileStorageClient fileStorageClient, FileCreate file)
         {
             if (sdk == null)
@@ -190,6 +197,14 @@ public static async Task<List<SimulatorModelRevision>> GetOrCreateSimulatorModel
             return new List<SimulatorModelRevision> { rev1, rev2 };
         }
 
+        public static async Task<List<SimulatorModelRevision>> GetOrCreateSimulatorModelRevisionsForCleanup(Client sdk, FileStorageClient fileStorageClient) {            
+            var revision = SimulatorModelRevisionCreateV3;            
+            var modelFile = await GetOrCreateFile(sdk, fileStorageClient, SimpleModelFileCreate3).ConfigureAwait(false);
+            revision.FileId = modelFile.Id;
+            var rev =await GetOrCreateSimulatorModelRevision(sdk, SimulatorModelCreate2, revision).ConfigureAwait(false);
+            return new List<SimulatorModelRevision> { rev };
+        }
+
         public static async Task<SimulatorRoutine> GetOrCreateSimulatorRoutine(Client sdk, SimulatorRoutineCreateCommandItem routine)
         {
             var routines = await sdk.Alpha.Simulators.ListSimulatorRoutinesAsync(
@@ -814,15 +829,27 @@ public static async Task<SimulatorRoutineRevision> GetOrCreateSimulatorRoutineRe
             Type = "OilWell",
         };
 
-        public static SimulatorModelRevisionCreate SimulatorModelRevisionCreateV1 = GenerateSimulatorModelRevisionCreate(TestModelExternalId, 1);
+        public static SimulatorModelCreate SimulatorModelCreate2 = new SimulatorModelCreate()
+        {
+            ExternalId = TestModelExternalId2,
+            Name = "Connector Test Model",
+            Description = "PETEX-Connector Test Model",
+            DataSetId = TestDataSetId,
+            SimulatorExternalId = TestSimulatorExternalId,
+            Type = "OilWell",
+        };
+
+        public static SimulatorModelRevisionCreate SimulatorModelRevisionCreateV1 = GenerateSimulatorModelRevisionCreate(SimulatorModelCreate, 1);
+
+        public static SimulatorModelRevisionCreate SimulatorModelRevisionCreateV2 = GenerateSimulatorModelRevisionCreate(SimulatorModelCreate, 2);
 
-        public static SimulatorModelRevisionCreate SimulatorModelRevisionCreateV2 = GenerateSimulatorModelRevisionCreate(TestModelExternalId, 2);
+        public static SimulatorModelRevisionCreate SimulatorModelRevisionCreateV3 = GenerateSimulatorModelRevisionCreate(SimulatorModelCreate2, 1);
 
-        public static SimulatorModelRevisionCreate GenerateSimulatorModelRevisionCreate(string externalId, int version = 1) {
+        public static SimulatorModelRevisionCreate GenerateSimulatorModelRevisionCreate(SimulatorModelCreate model, int version = 1) {
             return new SimulatorModelRevisionCreate()
             {
-                ExternalId = $"{externalId}-{version}",
-                ModelExternalId = SimulatorModelCreate.ExternalId,
+                ExternalId = $"{model.ExternalId}-{version}",
+                ModelExternalId = model.ExternalId,
                 Description = "integration test. can be deleted at any time. the test will recreate it.",
             };
         }

Cognite.Simulator.Tests/UtilsTests/ModelLibraryTest.cs

@@ -8,6 +8,7 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Xunit;
+using Moq;
 
 using Cognite.Extractor.StateStorage;
 using Cognite.Extractor.Utils;
@@ -17,6 +18,7 @@
 
 using Cognite.Simulator.Extensions;
 using Cognite.Simulator.Utils.Automation;
+using Microsoft.Extensions.DependencyInjection.Extensions;
 
 namespace Cognite.Simulator.Tests.UtilsTests
 {
@@ -32,6 +34,7 @@ public async Task TestModelLibrary()
             services.AddSingleton(SeedData.SimulatorCreate);
             services.AddSingleton<ModeLibraryTest>();
             services.AddSingleton<ModelParsingInfo>();
+            services.AddSingleton<IDateTimeProvider, SystemDateTimeProvider>();
             var loggerConfig = new LoggerConfig
             {
                 Console = new Extractor.Logging.ConsoleConfig
@@ -136,6 +139,141 @@ await modelLibTasks
             }
         }
 
+        [Fact]
+        public async Task TestModelLibraryCleanup(){
+            var utcNow = DateTime.UtcNow;
+            var mockDateTimeProvider = new Mock<IDateTimeProvider>();
+            mockDateTimeProvider.Setup(m => m.UtcNow).Returns(() => utcNow); // Dynamic value
+
+            var services = new ServiceCollection();
+            services.AddCogniteTestClient();
+            services.AddHttpClient<FileStorageClient>();
+            services.AddSingleton(SeedData.SimulatorCreate);
+            services.AddSingleton<ModeLibraryTest>();
+            services.AddSingleton<IDateTimeProvider>(mockDateTimeProvider.Object);
+
+            StateStoreConfig stateConfig = null;
+            using var provider = services.BuildServiceProvider();
+
+            var cdf = provider.GetRequiredService<Client>();
+
+            try
+            {
+                await SeedData.GetOrCreateSimulator(cdf, SeedData.SimulatorCreate).ConfigureAwait(false);
+                var fileStorageClient = provider.GetRequiredService<FileStorageClient>();
+                // prepopulate models and revisions in CDF
+                var revisionsRes = await SeedData.GetOrCreateSimulatorModelRevisions(cdf, fileStorageClient).ConfigureAwait(false);
+                var revisionsCleanup = await SeedData.GetOrCreateSimulatorModelRevisionsForCleanup(cdf, fileStorageClient).ConfigureAwait(false);
+
+                var revisions =
+                    revisionsRes.TakeLast(1)
+                    .Concat(revisionsCleanup)
+                    .ToList();
+
+                stateConfig = provider.GetRequiredService<StateStoreConfig>();
+                using var source = new CancellationTokenSource();
+
+                var lib = provider.GetRequiredService<ModeLibraryTest>();
+                await lib.Init(source.Token).ConfigureAwait(false);
+
+
+                var libState = (IReadOnlyDictionary<string, TestFileState>)lib._state;
+
+                Assert.NotEmpty(lib._state);
+                Assert.Equal(revisions.Count, lib._state.Count);
+
+                // Start the library update loop that download and parses the files, stop after 5 secs
+                using var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(source.Token);
+                var linkedToken = linkedTokenSource.Token;
+                linkedTokenSource.CancelAfter(TimeSpan.FromSeconds(10)); // should be enough time to download the file from CDF and parse it
+                var modelLibTasks = lib.GetRunTasks(linkedToken);
+                await modelLibTasks
+                    .RunAll(linkedTokenSource)
+                    .ConfigureAwait(false);
+
+                foreach (var revision in revisions)
+                {
+                    var modelInState = lib._state.GetValueOrDefault(revision.Id.ToString());
+                    Assert.NotNull(modelInState);
+                    Assert.Equal(revision.ExternalId, modelInState.ExternalId);
+                    Assert.False(modelInState.IsDeleted);
+                    Assert.False(string.IsNullOrEmpty(modelInState.FilePath));
+                    Assert.True(System.IO.File.Exists(modelInState.FilePath));
+                }
+
+                var modelRev1 = $"{SeedData.TestModelExternalId}-2";
+                var modelRev2 = $"{SeedData.TestModelExternalId2}-1";
+                var v1 = await lib.GetModelRevision(modelRev1).ConfigureAwait(false);
+                var v2 = await lib.GetModelRevision(modelRev2).ConfigureAwait(false);
+
+                Assert.True((v1.LastAccessTime - DateTime.UtcNow).TotalSeconds < 10);
+                Assert.True((v2.LastAccessTime - DateTime.UtcNow).TotalSeconds < 10);
+
+                // add 3 days to utcNow
+                utcNow = DateTime.UtcNow.AddDays(3);
+                await lib.Init(source.Token).ConfigureAwait(false);
+
+                lib.GetRunTasks(linkedToken);
+                    await modelLibTasks
+                        .RunAll(linkedTokenSource)
+                        .ConfigureAwait(false);
+
+                foreach (var revision in revisions)
+                {
+                    var modelInState = lib._state.GetValueOrDefault(revision.Id.ToString());
+                    Assert.NotNull(modelInState);
+                    Assert.Equal(revision.ExternalId, modelInState.ExternalId);
+                    Assert.False(modelInState.IsDeleted);
+                    Assert.True((utcNow - modelInState.LastAccessTime).TotalDays >= 3);
+                    Assert.False(string.IsNullOrEmpty(modelInState.FilePath));
+                    Assert.True(System.IO.File.Exists(modelInState.FilePath));
+                }
+
+                v2 = await lib.GetModelRevision(modelRev2).ConfigureAwait(false);
+                Assert.True((v2.LastAccessTime - utcNow).TotalSeconds < 10);
+
+                var rev1Id = revisionsRes.TakeLast(1).First().Id;
+                var rev1 = lib._state.GetValueOrDefault( rev1Id.ToString());
+                Assert.NotNull(rev1);
+                Assert.True((utcNow - rev1.LastAccessTime).TotalDays >= 3);
+
+
+                // add 5 days to utcNow
+                utcNow = utcNow.AddDays(5);
+                await lib.Init(source.Token).ConfigureAwait(false);
+
+                lib.GetRunTasks(linkedToken);
+                    await modelLibTasks
+                        .RunAll(linkedTokenSource)
+                        .ConfigureAwait(false);
+
+                rev1 = lib._state.GetValueOrDefault( rev1Id.ToString());
+                Assert.NotNull(rev1);
+                Assert.True(rev1.IsDeleted);
+                Assert.True((utcNow - rev1.LastAccessTime).TotalDays >= 7);
+                Assert.False(System.IO.File.Exists(rev1.FilePath));
+                Assert.True(System.IO.File.Exists(v2.FilePath));
+
+                v1 = await lib.GetModelRevision(modelRev1).ConfigureAwait(false);
+                Assert.False(v1.IsDeleted);
+                Assert.True((v1.LastAccessTime - utcNow).TotalSeconds < 10);
+                Assert.False(string.IsNullOrEmpty(v1.FilePath));
+                Assert.True(System.IO.File.Exists(v1.FilePath));
+            }
+            finally
+            {
+                if (Directory.Exists("./files"))
+                {
+                    Directory.Delete("./files", true);
+                }
+                if (stateConfig != null)
+                {
+                    StateUtils.DeleteLocalFile(stateConfig.Location);
+                }
+                await SeedData.DeleteSimulator(cdf, SeedData.SimulatorCreate.ExternalId);
+            }
+        }
+
         // This tests the situation when the model is marked as parsed before the library starts
         // In such cases the library should normally not re-parse the model
         [Fact]
@@ -149,6 +287,7 @@ public async Task TestModelLibraryWithReparse()
             services.AddSingleton<ModelParsingInfo>();
             services.AddSingleton<ScopedRemoteApiSink>();
             services.AddSingleton<DefaultConfig<AutomationConfig>>();
+            services.AddSingleton<IDateTimeProvider, SystemDateTimeProvider>();
             StateStoreConfig stateConfig = null;
             using var provider = services.BuildServiceProvider();
 
@@ -280,6 +419,7 @@ public async Task TestModelLibraryHotReload()
             services.AddSingleton<ScopedRemoteApiSink>();
             services.AddSingleton(SeedData.SimulatorCreate);
             services.AddSingleton<DefaultConfig<AutomationConfig>>();
+            services.AddSingleton<IDateTimeProvider, SystemDateTimeProvider>();
             StateStoreConfig stateConfig = null;
             using var provider = services.BuildServiceProvider();
 
@@ -324,7 +464,7 @@ await modelLibTasks
                 var modelExternalIdToDelete = revisions.First().ModelExternalId;
                 await SeedData.DeleteSimulatorModel(cdf, modelExternalIdToDelete).ConfigureAwait(false);
 
-                var revisionNewCreate = SeedData.GenerateSimulatorModelRevisionCreate("hot-reload", version: 1);
+                var revisionNewCreate = SeedData.GenerateSimulatorModelRevisionCreate(SeedData.SimulatorModelCreate, version: 1);
                 await SeedData.GetOrCreateSimulatorModelRevisionWithFile(cdf, fileStorageClient, SeedData.SimpleModelFileCreate, revisionNewCreate).ConfigureAwait(false);
 
                 var revisionNew = await SeedData.GetOrCreateSimulatorModelRevisionWithFile(cdf, fileStorageClient, SeedData.SimpleModelFileCreate, revisionNewCreate).ConfigureAwait(false);
@@ -545,6 +685,7 @@ public ModeLibraryTest(
             ILogger<ModeLibraryTest> logger,
             FileStorageClient downloadClient,
             SimulatorCreate simulatorDefinition,
+            IDateTimeProvider dateTimeProvider,
             IExtractionStateStore store = null) :
             base(
                 new ModelLibraryConfig
@@ -559,7 +700,9 @@ public ModeLibraryTest(
                 cdf,
                 logger,
                 downloadClient,
-                store)
+                dateTimeProvider,
+                store
+                )
         {
             _logger = logger;
         }

Cognite.Simulator.Tests/UtilsTests/SimulationRunnerTest.cs

@@ -124,6 +124,7 @@ bool debugLog
             services.AddSingleton<SampleSimulationRunner>();
             services.AddSingleton(SeedData.SimulatorCreate);
             services.AddSingleton<SampleSimulatorClient>();
+            services.AddSingleton<IDateTimeProvider, SystemDateTimeProvider>();
             services.AddSingleton(new ConnectorConfig
             {
                 NamePrefix = SeedData.TestIntegrationExternalId,

Cognite.Simulator.Utils/Configuration.cs

@@ -139,6 +139,12 @@
         /// Maximum number of download attempts before giving up. Default is 3.
         /// </summary>
         public int MaxDownloadAttempts { get; set; } = 3;
+
+        /// <summary>
+        /// Days to keep files in the library before deleting them, default is 7 days
+        /// Each time the model revision is accessed it will update the LastAccessedTime
+        /// </summary>
+        public int FilesTTL { get; set; } = 7;
     }
 
     /// <summary>
@@ -266,10 +272,10 @@
         public bool Enabled { get; set; }
     }
 
     public class DefaultConnectorConfig : ConnectorConfig
     {
         public ModelLibraryConfig ModelLibrary { get; set; }
         public RoutineLibraryConfig RoutineLibrary { get; set; }
     }
 
     public class DefaultConfig<TAutomationConfig> : BaseConfig 

Cognite.Simulator.Utils/DefaultClasses/DefaultModelLibrary.cs

@@ -36,13 +36,15 @@ public DefaultModelLibrary(
             ISimulatorClient<TModelStateBase, SimulatorRoutineRevision> simulatorClient,
             SimulatorCreate simulatorDefinition,
             FileStorageClient client,
+            IDateTimeProvider dateTimeProvider,
             IExtractionStateStore store = null) :
             base(
                 config.Connector.ModelLibrary,
                 simulatorDefinition,
                 cdf,
                 logger,
                 client,
+                dateTimeProvider,
                 store)
         {
             this.simulatorClient = simulatorClient;

Cognite.Simulator.Utils/FileState.cs

@@ -237,6 +237,36 @@ public int DownloadAttempts {
             }
         }
 
+
+        private DateTime _lastAccessTime;
+        /// <summary>
+        /// Last time the file was accessed
+        /// </summary>
+        public DateTime LastAccessTime {
+            get => _lastAccessTime;
+            set
+            {
+                if (value == _lastAccessTime) return;
+                LastTimeModified = DateTime.UtcNow;
+                _lastAccessTime = value;
+            }
+        }
+
+        private bool _isDeleted;
+        /// <summary>
+        /// If the file has been deleted
+        /// </summary>
+        public bool IsDeleted {
+            get => _isDeleted;
+            set
+            {
+                if (value == _isDeleted) return;
+                LastTimeModified = DateTime.UtcNow;
+                _isDeleted = value;
+            }
+        }
+
+
         /// <summary>
         /// Initialize this state using a data object from the state store
         /// </summary>
@@ -279,6 +309,7 @@ public virtual FileStatePoco GetPoco()
                 CdfId = CdfId,
                 UpdatedTime = UpdatedTime,
                 IsInDirectory = IsInDirectory,
+                isDeleted = IsDeleted,
                 ExternalId = ExternalId,
                 LogId = LogId,
                 FileExtension = FileExtension,
@@ -371,5 +402,17 @@ public class FileStatePoco : BaseStorableState
         /// </summary>
         [StateStoreProperty("download-attempts")]
         public int DownloadAttempts { get; set; }
+
+        /// <summary>
+        /// If the file has been deleted
+        /// </summary>
+        [StateStoreProperty("is-deleted")]
+        public bool isDeleted { get; set; }
+
+        /// <summary>
+        /// Last time the file was accessed
+        /// </summary>
+        [StateStoreProperty("last-access-time")]
+        public DateTime LastAccessTime { get; set; }
     }
 }