Merge pull request #181 from NicolasConstant/topic_autodelete-old-posts

Topic autodelete old posts
NicolasConstant · Mar 14, 2023 · 03a6c53 · 03a6c53
2 parents d80a001 + 5b93c2b
commit 03a6c53
Show file tree

Hide file tree

Showing 82 changed files with 1,572 additions and 174 deletions.
diff --git a/.github/workflows/dotnet-core.yml b/.github/workflows/dotnet-core.yml
@@ -16,7 +16,7 @@ jobs:
     - name: Setup .NET Core
       uses: actions/setup-dotnet@v1
       with:
-        dotnet-version: 3.1.101
+        dotnet-version: 6.0.x
     - name: Install dependencies
       run: dotnet restore
       working-directory: ${{env.working-directory}}

diff --git a/Dockerfile b/Dockerfile
@@ -1,11 +1,11 @@
 #See https://aka.ms/containerfastmode to understand how Visual Studio uses this Dockerfile to build your images for faster debugging.
 
-FROM mcr.microsoft.com/dotnet/aspnet:3.1-buster-slim AS base
+FROM mcr.microsoft.com/dotnet/aspnet:6.0-bullseye-slim AS base
 WORKDIR /app
 EXPOSE 80
 EXPOSE 443
 
-FROM mcr.microsoft.com/dotnet/sdk:3.1-buster AS publish
+FROM mcr.microsoft.com/dotnet/sdk:6.0-bullseye-slim AS publish
 COPY ./src/ ./src/
 RUN dotnet publish "/src/BirdsiteLive/BirdsiteLive.csproj" -c Release -o /app/publish
 RUN dotnet publish "/src/BSLManager/BSLManager.csproj" -r linux-x64 --self-contained true -p:PublishSingleFile=true -p:IncludeAllContentForSelfExtract=true -c Release -o /app/publish

diff --git a/VARIABLES.md b/VARIABLES.md
@@ -53,6 +53,7 @@ If both whitelisting and blacklisting are set, only the whitelisting will be act
 * `Instance:UserCacheCapacity` (default: 10000) set the caching limit of the Twitter User retrieval. Must be higher than the number of synchronized accounts on the instance.
 * `Instance:IpWhiteListing` IP Whitelisting (separated by `;`), prevent usage of the instance from other IPs than those provided (if provided).
 * `Instance:EnableXRealIpHeader` (default: false) Enable support of X-Real-IP Header to get the remote IP (useful when using reverse proxy).
+* `Instance:MaxTweetRetention` (default: 20, min: 1, max: 90) Number of days before synchronized tweets get deleted
 
 # Docker Compose full example
 

diff --git a/src/BSLManager/BSLManager.csproj b/src/BSLManager/BSLManager.csproj
@@ -2,15 +2,15 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Lamar" Version="5.0.3" />
-    <PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="5.0.0" />
-    <PackageReference Include="Terminal.Gui" Version="1.0.0-beta.11" />
+    <PackageReference Include="Lamar" Version="10.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="7.0.0" />
+    <PackageReference Include="Terminal.Gui" Version="1.9.0" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj b/src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj
@@ -6,8 +6,8 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
-    <PackageReference Include="System.Text.Json" Version="4.7.2" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
+    <PackageReference Include="System.Text.Json" Version="7.0.1" />
   </ItemGroup>
 
 </Project>
diff --git a/src/BirdsiteLive.ActivityPub/Models/Tombstone.cs b/src/BirdsiteLive.ActivityPub/Models/Tombstone.cs
@@ -0,0 +1,9 @@
+namespace BirdsiteLive.ActivityPub.Models
+{
+    public class Tombstone
+    {
+        public string id { get; set; }
+        public readonly string type = "Tombstone";
+        public string atomUrl { get; set; }
+    }
+}
diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs
@@ -18,5 +18,7 @@ public class InstanceSettings
         public int UserCacheCapacity { get; set; }
         public string IpWhiteListing { get; set; }
         public bool EnableXRealIpHeader { get; set; }
+
+        public int MaxTweetRetention { get; set; }
     }
 }
diff --git a/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj b/src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj
@@ -6,8 +6,8 @@
 
   <ItemGroup>
     <PackageReference Include="Asn1" Version="1.0.9" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
-    <PackageReference Include="Portable.BouncyCastle" Version="1.8.6.7" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
+    <PackageReference Include="Portable.BouncyCastle" Version="1.9.0" />
   </ItemGroup>
 
 </Project>
diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs
@@ -9,6 +9,7 @@
 using BirdsiteLive.ActivityPub.Converters;
 using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Settings;
+using BirdsiteLive.DAL.Models;
 using Microsoft.Extensions.Logging;
 using Newtonsoft.Json;
 
@@ -22,6 +23,7 @@ public interface IActivityPubService
         Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost,
             string targetInbox);
         Task DeleteUserAsync(string username, string targetHost, string targetInbox);
+        Task DeleteNoteAsync(SyncTweet tweet);
     }
 
     public class WebFinger
@@ -108,6 +110,30 @@ public async Task DeleteUserAsync(string username, string targetHost, string tar
             }
         }
 
+        public async Task DeleteNoteAsync(SyncTweet tweet)
+        {
+            var acct = tweet.Acct.ToLowerInvariant().Trim();
+
+            var actor = $"https://{_instanceSettings.Domain}/users/{acct}";
+            var noteId = $"https://{_instanceSettings.Domain}/users/{acct}/statuses/{tweet.TweetId}";
+
+            var delete = new ActivityDelete
+            {
+                context = "https://www.w3.org/ns/activitystreams",
+                id = $"{noteId}#delete",
+                type = "Delete",
+                actor = actor,
+                to = new[] { "https://www.w3.org/ns/activitystreams#Public" },
+                apObject = new Tombstone
+                {
+                    id = noteId,
+                    atomUrl = noteId
+                }
+            };
+
+            await PostDataAsync(delete, tweet.Host, actor, tweet.Inbox);
+        }
+
         public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox)
         {
             try

diff --git a/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj b/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj
@@ -5,7 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Http" Version="7.0.0" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs b/src/BirdsiteLive.Domain/BusinessUseCases/ProcessDeleteUser.cs
@@ -9,6 +9,7 @@ public interface IProcessDeleteUser
     {
         Task ExecuteAsync(Follower follower);
         Task ExecuteAsync(string followerUsername, string followerDomain);
+        Task ExecuteAsync(string actorId);
     }
 
     public class ProcessDeleteUser : IProcessDeleteUser
@@ -33,6 +34,15 @@ public async Task ExecuteAsync(string followerUsername, string followerDomain)
             await ExecuteAsync(follower);
         }
 
+        public async Task ExecuteAsync(string actorId)
+        {
+            // Get Follower and Twitter Users
+            var follower = await _followersDal.GetFollowerAsync(actorId);
+            if (follower == null) return;
+
+            await ExecuteAsync(follower);
+        }
+
         public async Task ExecuteAsync(Follower follower)
         {
             // Remove twitter users if no more followers

diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs
@@ -280,15 +280,21 @@ public async Task<bool> UndoFollowRequestedAsync(string signature, string method
         public async Task<bool> DeleteRequestedAsync(string signature, string method, string path, string queryString, Dictionary<string, string> requestHeaders,
             ActivityDelete activity, string body)
         {
-            // Validate
-            var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body);
-            if (!sigValidation.SignatureIsValidated) return false;
+            if (activity.apObject is string apObject)
+            {
+                if (!string.Equals(activity.actor.Trim(), apObject.Trim(), StringComparison.InvariantCultureIgnoreCase)) return true;
 
-            // Remove user and followings
-            var followerUserName = SigValidationResultExtractor.GetUserName(sigValidation);
-            var followerHost = SigValidationResultExtractor.GetHost(sigValidation);
+                try
+                {
+                    // Validate
+                    var sigValidation = await ValidateSignature(activity.actor, signature, method, path, queryString, requestHeaders, body);
+                    if (!sigValidation.SignatureIsValidated) return false;
+                }
+                catch (FollowerIsGoneException){}
 
-            await _processDeleteUser.ExecuteAsync(followerUserName, followerHost);
+                // Remove user and followings
+                await _processDeleteUser.ExecuteAsync(activity.actor.Trim());
+            }
 
             return true;
         }

diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj
@@ -1,14 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="5.0.0" />
-    <PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.11.1" />
+    <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
+    <PackageReference Include="System.Threading.Tasks.Dataflow" Version="7.0.0" />
   </ItemGroup>
 
   <ItemGroup>
@@ -19,6 +19,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <Folder Include="Processors\TweetsCleanUp\Base\" />
     <Folder Include="Tools\" />
   </ItemGroup>
 

diff --git a/...cts/IRefreshTwitterUserStatusProcessor.cs → ...ion/IRefreshTwitterUserStatusProcessor.cs b/...cts/IRefreshTwitterUserStatusProcessor.cs → ...ion/IRefreshTwitterUserStatusProcessor.cs
@@ -3,7 +3,7 @@
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface IRefreshTwitterUserStatusProcessor
     {

diff --git a/.../Contracts/IRetrieveFollowersProcessor.cs → ...Federation/IRetrieveFollowersProcessor.cs b/.../Contracts/IRetrieveFollowersProcessor.cs → ...Federation/IRetrieveFollowersProcessor.cs
@@ -3,7 +3,7 @@
 using System.Threading.Tasks;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface IRetrieveFollowersProcessor
     {

diff --git a/...ine/Contracts/IRetrieveTweetsProcessor.cs → ...ts/Federation/IRetrieveTweetsProcessor.cs b/...ine/Contracts/IRetrieveTweetsProcessor.cs → ...ts/Federation/IRetrieveTweetsProcessor.cs
@@ -3,7 +3,7 @@
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface IRetrieveTweetsProcessor
     {

diff --git a/...ntracts/IRetrieveTwitterUsersProcessor.cs → ...eration/IRetrieveTwitterUsersProcessor.cs b/...ntracts/IRetrieveTwitterUsersProcessor.cs → ...eration/IRetrieveTwitterUsersProcessor.cs
@@ -3,7 +3,7 @@
 using System.Threading.Tasks.Dataflow;
 using BirdsiteLive.DAL.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface IRetrieveTwitterUsersProcessor
     {

diff --git a/...ne/Contracts/ISaveProgressionProcessor.cs → ...s/Federation/ISaveProgressionProcessor.cs b/...ne/Contracts/ISaveProgressionProcessor.cs → ...s/Federation/ISaveProgressionProcessor.cs
@@ -2,7 +2,7 @@
 using System.Threading.Tasks;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface ISaveProgressionProcessor
     {

diff --git a/...tracts/ISendTweetsToFollowersProcessor.cs → ...ration/ISendTweetsToFollowersProcessor.cs b/...tracts/ISendTweetsToFollowersProcessor.cs → ...ration/ISendTweetsToFollowersProcessor.cs
@@ -2,7 +2,7 @@
 using System.Threading.Tasks;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Contracts
+namespace BirdsiteLive.Pipeline.Contracts.Federation
 {
     public interface ISendTweetsToFollowersProcessor
     {

diff --git a/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IDeleteTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IDeleteTweetsProcessor.cs
@@ -0,0 +1,11 @@
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.Pipeline.Models;
+
+namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp
+{
+    public interface IDeleteTweetsProcessor
+    {
+        Task<TweetToDelete> ProcessAsync(TweetToDelete tweet, CancellationToken ct);
+    }
+}
diff --git a/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IRetrieveTweetsToDeleteProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/IRetrieveTweetsToDeleteProcessor.cs
@@ -0,0 +1,12 @@
+using System.Threading;
+using System.Threading.Tasks;
+using System.Threading.Tasks.Dataflow;
+using BirdsiteLive.Pipeline.Models;
+
+namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp
+{
+    public interface IRetrieveTweetsToDeleteProcessor
+    {
+        Task GetTweetsAsync(BufferBlock<TweetToDelete> tweetsBufferBlock, CancellationToken ct);
+    }
+}
diff --git a/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/ISaveDeletedTweetStatusProcessor.cs b/src/BirdsiteLive.Pipeline/Contracts/TweetsCleanUp/ISaveDeletedTweetStatusProcessor.cs
@@ -0,0 +1,11 @@
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.Pipeline.Models;
+
+namespace BirdsiteLive.Pipeline.Contracts.TweetsCleanUp
+{
+    public interface ISaveDeletedTweetStatusProcessor
+    {
+        Task ProcessAsync(TweetToDelete tweetToDelete, CancellationToken ct);
+    }
+}
diff --git a/src/BirdsiteLive.Pipeline/Models/TweetToDelete.cs b/src/BirdsiteLive.Pipeline/Models/TweetToDelete.cs
@@ -0,0 +1,10 @@
+using BirdsiteLive.DAL.Models;
+
+namespace BirdsiteLive.Pipeline.Models
+{
+    public class TweetToDelete
+    {
+        public SyncTweet Tweet { get; set; }
+        public bool DeleteSuccessful { get; set; }
+    }
+}
diff --git a/...sors/RefreshTwitterUserStatusProcessor.cs → ...tion/RefreshTwitterUserStatusProcessor.cs b/...sors/RefreshTwitterUserStatusProcessor.cs → ...tion/RefreshTwitterUserStatusProcessor.cs
@@ -6,12 +6,12 @@
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Moderation.Actions;
-using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Contracts.Federation;
 using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
 
-namespace BirdsiteLive.Pipeline.Processors
+namespace BirdsiteLive.Pipeline.Processors.Federation
 {
     public class RefreshTwitterUserStatusProcessor : IRefreshTwitterUserStatusProcessor
     {

diff --git a/.../Processors/RetrieveFollowersProcessor.cs → .../Federation/RetrieveFollowersProcessor.cs b/.../Processors/RetrieveFollowersProcessor.cs → .../Federation/RetrieveFollowersProcessor.cs
@@ -2,10 +2,10 @@
 using System.Threading;
 using System.Threading.Tasks;
 using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Contracts.Federation;
 using BirdsiteLive.Pipeline.Models;
 
-namespace BirdsiteLive.Pipeline.Processors
+namespace BirdsiteLive.Pipeline.Processors.Federation
 {
     public class RetrieveFollowersProcessor : IRetrieveFollowersProcessor
     {

diff --git a/...ine/Processors/RetrieveTweetsProcessor.cs → ...ors/Federation/RetrieveTweetsProcessor.cs b/...ine/Processors/RetrieveTweetsProcessor.cs → ...ors/Federation/RetrieveTweetsProcessor.cs
@@ -5,14 +5,14 @@
 using System.Threading.Tasks;
 using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Contracts.Federation;
 using BirdsiteLive.Pipeline.Models;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Models;
 using Microsoft.Extensions.Logging;
 using Tweetinvi.Models;
 
-namespace BirdsiteLive.Pipeline.Processors
+namespace BirdsiteLive.Pipeline.Processors.Federation
 {
     public class RetrieveTweetsProcessor : IRetrieveTweetsProcessor
     {
@@ -64,7 +64,7 @@ public async Task<UserWithDataToSync[]> ProcessAsync(UserWithDataToSync[] syncTw
         private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user)
         {
             var tweets = new ExtractedTweet[0];
-            
+
             try
             {
                 if (user.LastTweetPostedId == -1)