Purge rgpd after 5 years

app/config/TaskConfiguration.scala

@@ -11,6 +11,7 @@ case class TaskConfiguration(
     reportClosure: ReportClosureTaskConfiguration,
     orphanReportFileDeletion: OrphanReportFileDeletionTaskConfiguration,
     oldReportExportDeletion: OldReportExportDeletionTaskConfiguration,
+    oldReportsRgpdDeletion: OldReportsRgpdDeletionTaskConfiguration,
     reportReminders: ReportRemindersTaskConfiguration,
     inactiveAccounts: InactiveAccountsTaskConfiguration,
     companyUpdate: CompanyUpdateTaskConfiguration,
@@ -45,6 +46,10 @@ case class OldReportExportDeletionTaskConfiguration(
     startTime: LocalTime
 )
 
+case class OldReportsRgpdDeletionTaskConfiguration(
+    startTime: LocalTime
+)
+
 case class ReportRemindersTaskConfiguration(
     startTime: LocalTime,
     intervalInHours: FiniteDuration,

app/loader/SignalConsoApplicationLoader.scala

@@ -111,6 +111,7 @@ import tasks.account.InactiveDgccrfAccountReminderTask
 import tasks.account.InactiveDgccrfAccountRemoveTask
 import tasks.company._
 import tasks.report.OldReportExportDeletionTask
+import tasks.report.OldReportsRgpdDeletionTask
 import tasks.report.OrphanReportFileDeletionTask
 import tasks.report.ReportClosureTask
 import tasks.report.ReportNotificationTask
@@ -484,6 +485,14 @@ class SignalConsoComponents(
       websiteApiService
     )
 
+  val rgpdOrchestrator = new RgpdOrchestrator(
+    reportConsumerReviewOrchestrator,
+    engagementOrchestrator,
+    reportRepository,
+    reportFileOrchestrator,
+    eventRepository
+  )
+
   val reportAdminActionOrchestrator = new ReportAdminActionOrchestrator(
     mailService,
     reportConsumerReviewOrchestrator,
@@ -493,6 +502,7 @@ class SignalConsoComponents(
     companyRepository,
     eventRepository,
     companiesVisibilityOrchestrator,
+    rgpdOrchestrator,
     messagesApi
   )
 
@@ -541,6 +551,14 @@ class SignalConsoComponents(
     taskRepository
   )
 
+  val oldReportsRgpdDeletionTask = new OldReportsRgpdDeletionTask(
+    actorSystem,
+    reportRepository,
+    rgpdOrchestrator,
+    taskConfiguration,
+    taskRepository
+  )
+
   val reportReminderTask = new ReportRemindersTask(
     actorSystem,
     reportRepository,
@@ -875,6 +893,7 @@ class SignalConsoComponents(
     reportReminderTask.schedule()
     orphanReportFileDeletionTask.schedule()
     oldReportExportDeletionTask.schedule()
+    oldReportsRgpdDeletionTask.schedule()
     if (applicationConfiguration.task.probe.active) {
       probeOrchestrator.scheduleProbeTasks()
     }

app/orchestrators/ReportAdminActionOrchestrator.scala

@@ -10,7 +10,6 @@ import io.scalaland.chimney.dsl._
 import models._
 import models.company.Company
 import models.event.Event
-import models.report.ReportStatus.SuppressionRGPD
 import models.report.ReportStatus.Transmis
 import models.report._
 import models.report.delete.ReportAdminAction
@@ -20,7 +19,6 @@ import play.api.Logger
 import play.api.i18n.MessagesApi
 import play.api.libs.json.Json
 import repositories.company.CompanyRepositoryInterface
-import repositories.event.EventFilter
 import repositories.event.EventRepositoryInterface
 import repositories.report.ReportRepositoryInterface
 import services.emails.EmailDefinitionsConsumer.ConsumerProResponseNotificationOnAdminCompletion
@@ -30,9 +28,6 @@ import services.emails.EmailDefinitionsPro.ProResponseAcknowledgmentOnAdminCompl
 import services.emails.MailService
 import utils.Constants
 import utils.Constants.ActionEvent._
-import utils.Constants.ActionEvent
-import utils.Constants.EventType
-import utils.EmailAddress.EmptyEmailAddress
 import utils.Logs.RichLogger
 import utils.SIREN.fromSIRET
 
@@ -52,6 +47,7 @@ class ReportAdminActionOrchestrator(
     companyRepository: CompanyRepositoryInterface,
     eventRepository: EventRepositoryInterface,
     companiesVisibilityOrchestrator: CompaniesVisibilityOrchestrator,
+    rgpdOrchestrator: RgpdOrchestrator,
     messagesApi: MessagesApi
 )(implicit val executionContext: ExecutionContext) {
   val logger = Logger(this.getClass)
@@ -171,36 +167,13 @@ class ReportAdminActionOrchestrator(
       report: Report,
       user: User,
       reportAdminCompletionDetails: ReportAdminCompletionDetails
-  ): Future[Report] = {
-    val emptiedReport = report.copy(
-      firstName = "",
-      lastName = "",
-      consumerPhone = report.consumerPhone.map(_ => ""),
-      consumerReferenceNumber = report.consumerReferenceNumber.map(_ => ""),
-      email = EmptyEmailAddress,
-      details = List.empty,
-      status = SuppressionRGPD
-    )
+  ): Future[Report] =
     for {
-      _ <- reportRepository.update(emptiedReport.id, emptiedReport)
-      proEvents <- eventRepository.getEvents(
-        reportId = emptiedReport.id,
-        filter = EventFilter(eventType = Some(EventType.PRO), action = Some(ActionEvent.REPORT_PRO_RESPONSE))
-      )
-      _ <- proEvents.traverse { event =>
-        val emptiedDetails = event.details
-          .as[ExistingReportResponse]
-          .copy(fileIds = List.empty, consumerDetails = "", dgccrfDetails = None)
-        eventRepository.update(event.id, event.copy(details = Json.toJson(emptiedDetails)))
-      }
-      _            <- reportConsumerReviewOrchestrator.deleteDetails(emptiedReport.id)
-      _            <- engagementOrchestrator.deleteDetails(emptiedReport.id)
-      _            <- reportFileOrchestrator.removeFromReportId(emptiedReport.id)
-      maybeCompany <- report.companySiret.map(companyRepository.findBySiret).flatSequence
+      emptiedReport <- rgpdOrchestrator.deleteRGPD(report)
+      maybeCompany  <- report.companySiret.map(companyRepository.findBySiret).flatSequence
       _ <- createAdminDeletionReportEvent(report.companyId, user, RGPD_DELETE_REQUEST, reportAdminCompletionDetails)
       _ <- mailService.send(ConsumerReportDeletionConfirmation.Email(report, maybeCompany, messagesApi))
     } yield emptiedReport
-  }
 
   private def deleteReportFromConsumerRequest(
       id: UUID,

app/orchestrators/RgpdOrchestrator.scala

@@ -0,0 +1,59 @@
+package orchestrators
+
+import cats.implicits.toTraverseOps
+import models.report.ReportStatus.SuppressionRGPD
+import models.report._
+import play.api.Logger
+import play.api.libs.json.Json
+import repositories.event.EventFilter
+import repositories.event.EventRepositoryInterface
+import repositories.report.ReportRepositoryInterface
+import utils.Constants.ActionEvent
+import utils.Constants.EventType
+import utils.EmailAddress.EmptyEmailAddress
+
+import scala.concurrent.ExecutionContext
+import scala.concurrent.Future
+
+class RgpdOrchestrator(
+    reportConsumerReviewOrchestrator: ReportConsumerReviewOrchestrator,
+    engagementOrchestrator: EngagementOrchestrator,
+    reportRepository: ReportRepositoryInterface,
+    reportFileOrchestrator: ReportFileOrchestrator,
+    eventRepository: EventRepositoryInterface
+)(implicit val executionContext: ExecutionContext) {
+  val logger = Logger(this.getClass)
+
+  def deleteRGPD(
+      report: Report
+  ): Future[Report] = {
+    logger.info(s"Emptying report ${report.id} for RGPD")
+    val emptiedReport = report.copy(
+      firstName = "",
+      lastName = "",
+      consumerPhone = report.consumerPhone.map(_ => ""),
+      consumerReferenceNumber = report.consumerReferenceNumber.map(_ => ""),
+      email = EmptyEmailAddress,
+      details = List.empty,
+      status = SuppressionRGPD
+    )
+    for {
+      _ <- reportRepository.update(emptiedReport.id, emptiedReport)
+      proEvents <- eventRepository.getEvents(
+        reportId = emptiedReport.id,
+        filter = EventFilter(eventType = Some(EventType.PRO), action = Some(ActionEvent.REPORT_PRO_RESPONSE))
+      )
+      _ <- proEvents.traverse { event =>
+        val emptiedDetails = event.details
+          .as[ExistingReportResponse]
+          .copy(fileIds = List.empty, consumerDetails = "", dgccrfDetails = None)
+        eventRepository.update(event.id, event.copy(details = Json.toJson(emptiedDetails)))
+      }
+      _ <- reportConsumerReviewOrchestrator.deleteDetails(emptiedReport.id)
+      _ <- engagementOrchestrator.deleteDetails(emptiedReport.id)
+      _ <- reportFileOrchestrator.removeFromReportId(emptiedReport.id)
+      _ = logger.info(s"Report ${report.id} was emptied")
+    } yield emptiedReport
+  }
+
+}

app/repositories/report/ReportRepository.scala

@@ -7,6 +7,7 @@ import models._
 import models.barcode.BarcodeProduct
 import models.company.Company
 import models.report.ReportResponseType.ACCEPTED
+import models.report.ReportStatus.SuppressionRGPD
 import models.report._
 import models.report.reportmetadata.ReportMetadata
 import models.report.reportmetadata.ReportWithMetadata
@@ -404,6 +405,14 @@ class ReportRepository(override val dbConfig: DatabaseConfig[JdbcProfile])(impli
         .result
     )
 
+  def getOldReportsNotRgpdDeleted(createdBefore: OffsetDateTime): Future[List[Report]] = db.run(
+    table
+      .filter(_.creationDate < createdBefore)
+      .filter(_.status =!= SuppressionRGPD.entryName)
+      .to[List]
+      .result
+  )
+
   def getPendingReports(companiesIds: List[UUID]): Future[List[Report]] = db
     .run(
       table

app/repositories/report/ReportRepositoryInterface.scala

@@ -24,7 +24,6 @@ trait ReportRepositoryInterface extends CRUDRepositoryInterface[Report] {
   def streamReports: Source[Report, NotUsed]
 
   def streamAll: DatabasePublisher[((Report, Option[Company]), Option[BarcodeProduct])]
-
   def cloudWord(companyId: UUID): Future[List[ReportWordOccurrence]]
 
   def findSimilarReportList(
@@ -78,6 +77,7 @@ trait ReportRepositoryInterface extends CRUDRepositoryInterface[Report] {
 
   def getByStatusAndExpired(status: List[ReportStatus], now: OffsetDateTime): Future[List[Report]]
 
+  def getOldReportsNotRgpdDeleted(createdBefore: OffsetDateTime): Future[List[Report]]
   def getPendingReports(companiesIds: List[UUID]): Future[List[Report]]
 
   def getPhoneReports(start: Option[LocalDate], end: Option[LocalDate]): Future[List[Report]]

app/tasks/report/OldReportsRgpdDeletionTask.scala

@@ -0,0 +1,52 @@
+package tasks.report
+
+import config.TaskConfiguration
+import org.apache.pekko.actor.ActorSystem
+import repositories.report.ReportRepositoryInterface
+import repositories.tasklock.TaskRepositoryInterface
+import tasks.ScheduledTask
+import tasks.model.TaskSettings.DailyTaskSettings
+import controllers.error.AppError.ServerError
+import orchestrators.RgpdOrchestrator
+
+import java.time.OffsetDateTime
+import scala.concurrent.ExecutionContext
+import scala.concurrent.Future
+
+class OldReportsRgpdDeletionTask(
+    actorSystem: ActorSystem,
+    reportRepository: ReportRepositoryInterface,
+    rgpdOrchestrator: RgpdOrchestrator,
+    taskConfiguration: TaskConfiguration,
+    taskRepository: TaskRepositoryInterface
+)(implicit val executionContext: ExecutionContext)
+    extends ScheduledTask(9, "old_reports_rgpd_deletion_task", taskRepository, actorSystem, taskConfiguration) {
+
+  override val taskSettings = DailyTaskSettings(startTime = taskConfiguration.oldReportsRgpdDeletion.startTime)
+
+  // The maximum number of reports created in single day so far is 2335 on 2023-05-23
+  // So we shouldn't have to delete more than that each day
+  private val maxReportsAllowedToDelete = 2500
+
+  override def runTask(): Future[Unit] = {
+    val createdBefore = OffsetDateTime.now().minusYears(5)
+    for {
+      veryOldReports <- reportRepository.getOldReportsNotRgpdDeleted(createdBefore)
+      _ = if (veryOldReports.length > maxReportsAllowedToDelete) {
+        // Safety : if we had some date-related bug in this task, it could empty our whole database!
+        throw ServerError(
+          s"Rgpd deletion failed, it was going to delete ${veryOldReports.length} reports!"
+        )
+      }
+      _ = logger.info(s"Found ${veryOldReports.length} that need to be emptied for RGPD (older than ${createdBefore})")
+      _ <- veryOldReports.foldLeft(Future.successful(())) { case (previous, report) =>
+        for {
+          _ <- previous
+          _ <- rgpdOrchestrator.deleteRGPD(report)
+        } yield ()
+      }
+
+    } yield ()
+  }
+
+}

conf/common/task.conf

@@ -58,6 +58,11 @@ task {
     start-time = ${?OLD_REPORT_EXPORT_DELETION_TASK_START_TIME}
   }
 
+  old-reports-rgpd-deletion {
+    start-time = "07:30:00"
+    start-time = ${?OLD_REPORTS_RGPD_DELETION_TASK_START_TIME}
+  }
+
   sample-data {
     active = false
     active = ${?SAMPLE_DATA_GENERATION_TASK_ACTIVE}

test/controllers/report/ReportRepositoryMock.scala

@@ -111,4 +111,6 @@ class ReportRepositoryMock(database: mutable.Map[UUID, Report] = mutable.Map.emp
   def streamAll: DatabasePublisher[((Report, Option[Company]), Option[BarcodeProduct])] = ???
 
   override def streamReports: Source[Report, NotUsed] = ???
+
+  override def getOldReportsNotRgpdDeleted(createdBefore: OffsetDateTime): Future[List[Report]] = ???
 }

test/tasks/report/ReportRemindersTaskUnitSpec.scala

@@ -49,6 +49,7 @@ class ReportRemindersTaskUnitSpec extends Specification with FutureMatchers {
     reportClosure = null,
     orphanReportFileDeletion = null,
     oldReportExportDeletion = null,
+    oldReportsRgpdDeletion = null,
     reportReminders = ReportRemindersTaskConfiguration(
       startTime = LocalTime.of(2, 0),
       intervalInHours = 1.day,