set tweet retention delay in settings

VARIABLES.md

@@ -53,6 +53,7 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act
 * `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance.
 * `Instance:IpWhiteListing` IP Whitelisting (separated by `;`), prevent usage of the instance from other IPs than those provided (if provided).
 * `Instance:EnableXRealIpHeader` (default: false) Enable support of X-Real-IP Header to get the remote IP (useful when using reverse proxy).
+* `Instance:MaxTweetRetention` (default: 20, min: 1, max: 90) Number of days before synchronized tweets get deleted
 
 # Docker Compose full example
 

src/BirdsiteLive.Common/Settings/InstanceSettings.cs

@@ -18,5 +18,7 @@ public class InstanceSettings
         public int UserCacheCapacity { get; set; }
         public string IpWhiteListing { get; set; }
         public bool EnableXRealIpHeader { get; set; }
+
+        public int MaxTweetRetention { get; set; }
     }
 }

src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj

@@ -19,7 +19,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <Folder Include="Processors\TweetsCleanUp\" />
+    <Folder Include="Processors\TweetsCleanUp\Base\" />
     <Folder Include="Tools\" />
   </ItemGroup>
 

src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/Base/RetentionBase.cs

@@ -0,0 +1,16 @@
+using BirdsiteLive.Common.Settings;
+using System;
+
+namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base
+{
+    public class RetentionBase
+    {
+        protected int GetRetentionTime(InstanceSettings settings)
+        {
+            var retentionTime = Math.Abs(settings.MaxTweetRetention);
+            if (retentionTime < 1) retentionTime = 1;
+            if (retentionTime > 90) retentionTime = 90;
+            return retentionTime;
+        }
+    }
+}

src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/RetrieveTweetsToDeleteProcessor.cs

@@ -3,20 +3,24 @@
 using System.Threading;
 using System.Threading.Tasks;
 using System.Threading.Tasks.Dataflow;
+using BirdsiteLive.Common.Settings;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
 using BirdsiteLive.Pipeline.Models;
+using BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base;
 
 namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
 {
-    public class RetrieveTweetsToDeleteProcessor : IRetrieveTweetsToDeleteProcessor
+    public class RetrieveTweetsToDeleteProcessor : RetentionBase, IRetrieveTweetsToDeleteProcessor
     {
         private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal;
+        private readonly InstanceSettings _instanceSettings;
 
         #region Ctor
-        public RetrieveTweetsToDeleteProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal)
+        public RetrieveTweetsToDeleteProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal, InstanceSettings instanceSettings)
         {
             _syncTweetsPostgresDal = syncTweetsPostgresDal;
+            _instanceSettings = instanceSettings;
         }
         #endregion
 
@@ -29,7 +33,9 @@ public async Task GetTweetsAsync(BufferBlock<TweetToDelete> tweetsBufferBlock, C
                 ct.ThrowIfCancellationRequested();
 
                 var now = DateTime.UtcNow;
-                var from = now.AddDays(-20);
+
+
+                var from = now.AddDays(-GetRetentionTime(_instanceSettings));
                 var dbBrowsingEnded = false;
                 var lastId = -1L;
 

src/BirdsiteLive.Pipeline/Processors/TweetsCleanUp/SaveDeletedTweetStatusProcessor.cs

@@ -1,26 +1,32 @@
 using System;
 using System.Threading;
 using System.Threading.Tasks;
+using BirdsiteLive.Common.Settings;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.Pipeline.Contracts.TweetsCleanUp;
 using BirdsiteLive.Pipeline.Models;
+using BirdsiteLive.Pipeline.Processors.TweetsCleanUp.Base;
 
 namespace BirdsiteLive.Pipeline.Processors.TweetsCleanUp
 {
-    public class SaveDeletedTweetStatusProcessor : ISaveDeletedTweetStatusProcessor
+    public class SaveDeletedTweetStatusProcessor : RetentionBase, ISaveDeletedTweetStatusProcessor
     {
         private readonly ISyncTweetsPostgresDal _syncTweetsPostgresDal;
+        private readonly InstanceSettings _instanceSettings;
 
         #region Ctor
-        public SaveDeletedTweetStatusProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal)
+        public SaveDeletedTweetStatusProcessor(ISyncTweetsPostgresDal syncTweetsPostgresDal, InstanceSettings instanceSettings)
         {
             _syncTweetsPostgresDal = syncTweetsPostgresDal;
+            _instanceSettings = instanceSettings;
         }
         #endregion
 
         public async Task ProcessAsync(TweetToDelete tweetToDelete, CancellationToken ct)
         {
-            var highLimitDate = DateTime.UtcNow.AddDays(-40); //TODO get settings value
+            var retentionTime = GetRetentionTime(_instanceSettings);
+            retentionTime += 20; // Delay until last retry
+            var highLimitDate = DateTime.UtcNow.AddDays(-retentionTime);
             if (tweetToDelete.DeleteSuccessful || tweetToDelete.Tweet.PublishedAt < highLimitDate)
             {
                 await _syncTweetsPostgresDal.DeleteTweetAsync(tweetToDelete.Tweet.Id);

src/BirdsiteLive/appsettings.json

@@ -25,7 +25,8 @@
     "SensitiveTwitterAccounts": null,
     "FailingTwitterUserCleanUpThreshold": 700,
     "FailingFollowerCleanUpThreshold": 30000,
-    "UserCacheCapacity": 10000 
+    "UserCacheCapacity": 10000,
+    "MaxTweetRetention": 20
   },
   "Db": {
     "Type": "postgres",