From 6221274440ac744e9d0c05df2632c831905792e1 Mon Sep 17 00:00:00 2001 From: eletallbetagouv <107104509+eletallbetagouv@users.noreply.github.com> Date: Mon, 14 Oct 2024 22:27:02 +0300 Subject: [PATCH 1/4] draft suppression rgpd --- app/config/TaskConfiguration.scala | 5 ++ app/loader/SignalConsoApplicationLoader.scala | 19 +++++++ .../ReportAdminActionOrchestrator.scala | 35 ++---------- app/orchestrators/RgpdOrchestrator.scala | 57 +++++++++++++++++++ .../report/ReportRepository.scala | 7 +++ .../report/ReportRepositoryInterface.scala | 2 +- .../report/OldReportsRgpdDeletionTask.scala | 51 +++++++++++++++++ conf/common/task.conf | 5 ++ 8 files changed, 149 insertions(+), 32 deletions(-) create mode 100644 app/orchestrators/RgpdOrchestrator.scala create mode 100644 app/tasks/report/OldReportsRgpdDeletionTask.scala diff --git a/app/config/TaskConfiguration.scala b/app/config/TaskConfiguration.scala index fb5e92ce..e6ef6cc1 100644 --- a/app/config/TaskConfiguration.scala +++ b/app/config/TaskConfiguration.scala @@ -11,6 +11,7 @@ case class TaskConfiguration( reportClosure: ReportClosureTaskConfiguration, orphanReportFileDeletion: OrphanReportFileDeletionTaskConfiguration, oldReportExportDeletion: OldReportExportDeletionTaskConfiguration, + oldReportsRgpdDeletion: OldReportsRgpdDeletionTaskConfiguration, reportReminders: ReportRemindersTaskConfiguration, inactiveAccounts: InactiveAccountsTaskConfiguration, companyUpdate: CompanyUpdateTaskConfiguration, @@ -45,6 +46,10 @@ case class OldReportExportDeletionTaskConfiguration( startTime: LocalTime ) +case class OldReportsRgpdDeletionTaskConfiguration( + startTime: LocalTime +) + case class ReportRemindersTaskConfiguration( startTime: LocalTime, intervalInHours: FiniteDuration, diff --git a/app/loader/SignalConsoApplicationLoader.scala b/app/loader/SignalConsoApplicationLoader.scala index 1b6b5528..5272a6df 100644 --- a/app/loader/SignalConsoApplicationLoader.scala +++ b/app/loader/SignalConsoApplicationLoader.scala @@ -111,6 +111,7 @@ import tasks.account.InactiveDgccrfAccountReminderTask import tasks.account.InactiveDgccrfAccountRemoveTask import tasks.company._ import tasks.report.OldReportExportDeletionTask +import tasks.report.OldReportsRgpdDeletionTask import tasks.report.OrphanReportFileDeletionTask import tasks.report.ReportClosureTask import tasks.report.ReportNotificationTask @@ -484,6 +485,14 @@ class SignalConsoComponents( websiteApiService ) + val rgpdOrchestrator = new RgpdOrchestrator( + reportConsumerReviewOrchestrator, + engagementOrchestrator, + reportRepository, + reportFileOrchestrator, + eventRepository + ) + val reportAdminActionOrchestrator = new ReportAdminActionOrchestrator( mailService, reportConsumerReviewOrchestrator, @@ -493,6 +502,7 @@ class SignalConsoComponents( companyRepository, eventRepository, companiesVisibilityOrchestrator, + rgpdOrchestrator, messagesApi ) @@ -541,6 +551,14 @@ class SignalConsoComponents( taskRepository ) + val oldReportsRgpdDeletionTask = new OldReportsRgpdDeletionTask( + actorSystem, + reportRepository, + rgpdOrchestrator, + taskConfiguration, + taskRepository + ) + val reportReminderTask = new ReportRemindersTask( actorSystem, reportRepository, @@ -875,6 +893,7 @@ class SignalConsoComponents( reportReminderTask.schedule() orphanReportFileDeletionTask.schedule() oldReportExportDeletionTask.schedule() + oldReportsRgpdDeletionTask.schedule() if (applicationConfiguration.task.probe.active) { probeOrchestrator.scheduleProbeTasks() } diff --git a/app/orchestrators/ReportAdminActionOrchestrator.scala b/app/orchestrators/ReportAdminActionOrchestrator.scala index e603d98f..a2939905 100644 --- a/app/orchestrators/ReportAdminActionOrchestrator.scala +++ b/app/orchestrators/ReportAdminActionOrchestrator.scala @@ -10,7 +10,6 @@ import io.scalaland.chimney.dsl._ import models._ import models.company.Company import models.event.Event -import models.report.ReportStatus.SuppressionRGPD import models.report.ReportStatus.Transmis import models.report._ import models.report.delete.ReportAdminAction @@ -20,7 +19,6 @@ import play.api.Logger import play.api.i18n.MessagesApi import play.api.libs.json.Json import repositories.company.CompanyRepositoryInterface -import repositories.event.EventFilter import repositories.event.EventRepositoryInterface import repositories.report.ReportRepositoryInterface import services.emails.EmailDefinitionsConsumer.ConsumerProResponseNotificationOnAdminCompletion @@ -30,9 +28,6 @@ import services.emails.EmailDefinitionsPro.ProResponseAcknowledgmentOnAdminCompl import services.emails.MailService import utils.Constants import utils.Constants.ActionEvent._ -import utils.Constants.ActionEvent -import utils.Constants.EventType -import utils.EmailAddress.EmptyEmailAddress import utils.Logs.RichLogger import utils.SIREN.fromSIRET @@ -52,6 +47,7 @@ class ReportAdminActionOrchestrator( companyRepository: CompanyRepositoryInterface, eventRepository: EventRepositoryInterface, companiesVisibilityOrchestrator: CompaniesVisibilityOrchestrator, + rgpdOrchestrator: RgpdOrchestrator, messagesApi: MessagesApi )(implicit val executionContext: ExecutionContext) { val logger = Logger(this.getClass) @@ -171,36 +167,13 @@ class ReportAdminActionOrchestrator( report: Report, user: User, reportAdminCompletionDetails: ReportAdminCompletionDetails - ): Future[Report] = { - val emptiedReport = report.copy( - firstName = "", - lastName = "", - consumerPhone = report.consumerPhone.map(_ => ""), - consumerReferenceNumber = report.consumerReferenceNumber.map(_ => ""), - email = EmptyEmailAddress, - details = List.empty, - status = SuppressionRGPD - ) + ): Future[Report] = for { - _ <- reportRepository.update(emptiedReport.id, emptiedReport) - proEvents <- eventRepository.getEvents( - reportId = emptiedReport.id, - filter = EventFilter(eventType = Some(EventType.PRO), action = Some(ActionEvent.REPORT_PRO_RESPONSE)) - ) - _ <- proEvents.traverse { event => - val emptiedDetails = event.details - .as[ExistingReportResponse] - .copy(fileIds = List.empty, consumerDetails = "", dgccrfDetails = None) - eventRepository.update(event.id, event.copy(details = Json.toJson(emptiedDetails))) - } - _ <- reportConsumerReviewOrchestrator.deleteDetails(emptiedReport.id) - _ <- engagementOrchestrator.deleteDetails(emptiedReport.id) - _ <- reportFileOrchestrator.removeFromReportId(emptiedReport.id) - maybeCompany <- report.companySiret.map(companyRepository.findBySiret).flatSequence + emptiedReport <- rgpdOrchestrator.deleteRGPD(report) + maybeCompany <- report.companySiret.map(companyRepository.findBySiret).flatSequence _ <- createAdminDeletionReportEvent(report.companyId, user, RGPD_DELETE_REQUEST, reportAdminCompletionDetails) _ <- mailService.send(ConsumerReportDeletionConfirmation.Email(report, maybeCompany, messagesApi)) } yield emptiedReport - } private def deleteReportFromConsumerRequest( id: UUID, diff --git a/app/orchestrators/RgpdOrchestrator.scala b/app/orchestrators/RgpdOrchestrator.scala new file mode 100644 index 00000000..41afe006 --- /dev/null +++ b/app/orchestrators/RgpdOrchestrator.scala @@ -0,0 +1,57 @@ +package orchestrators + +import cats.implicits.toTraverseOps +import models.report.ReportStatus.SuppressionRGPD +import models.report._ +import play.api.Logger +import play.api.libs.json.Json +import repositories.event.EventFilter +import repositories.event.EventRepositoryInterface +import repositories.report.ReportRepositoryInterface +import utils.Constants.ActionEvent +import utils.Constants.EventType +import utils.EmailAddress.EmptyEmailAddress + +import scala.concurrent.ExecutionContext +import scala.concurrent.Future + +class RgpdOrchestrator( + reportConsumerReviewOrchestrator: ReportConsumerReviewOrchestrator, + engagementOrchestrator: EngagementOrchestrator, + reportRepository: ReportRepositoryInterface, + reportFileOrchestrator: ReportFileOrchestrator, + eventRepository: EventRepositoryInterface, +)(implicit val executionContext: ExecutionContext) { + val logger = Logger(this.getClass) + + def deleteRGPD( + report: Report + ): Future[Report] = { + val emptiedReport = report.copy( + firstName = "", + lastName = "", + consumerPhone = report.consumerPhone.map(_ => ""), + consumerReferenceNumber = report.consumerReferenceNumber.map(_ => ""), + email = EmptyEmailAddress, + details = List.empty, + status = SuppressionRGPD + ) + for { + _ <- reportRepository.update(emptiedReport.id, emptiedReport) + proEvents <- eventRepository.getEvents( + reportId = emptiedReport.id, + filter = EventFilter(eventType = Some(EventType.PRO), action = Some(ActionEvent.REPORT_PRO_RESPONSE)) + ) + _ <- proEvents.traverse { event => + val emptiedDetails = event.details + .as[ExistingReportResponse] + .copy(fileIds = List.empty, consumerDetails = "", dgccrfDetails = None) + eventRepository.update(event.id, event.copy(details = Json.toJson(emptiedDetails))) + } + _ <- reportConsumerReviewOrchestrator.deleteDetails(emptiedReport.id) + _ <- engagementOrchestrator.deleteDetails(emptiedReport.id) + _ <- reportFileOrchestrator.removeFromReportId(emptiedReport.id) + } yield emptiedReport + } + +} diff --git a/app/repositories/report/ReportRepository.scala b/app/repositories/report/ReportRepository.scala index b1a31f3f..0a2344d7 100644 --- a/app/repositories/report/ReportRepository.scala +++ b/app/repositories/report/ReportRepository.scala @@ -404,6 +404,13 @@ class ReportRepository(override val dbConfig: DatabaseConfig[JdbcProfile])(impli .result ) + def getOldReports(createdBefore: OffsetDateTime): Future[List[Report]] = db.run( + table + .filter(_.creationDate < createdBefore) + .to[List] + .result + ) + def getPendingReports(companiesIds: List[UUID]): Future[List[Report]] = db .run( table diff --git a/app/repositories/report/ReportRepositoryInterface.scala b/app/repositories/report/ReportRepositoryInterface.scala index 17240866..9ef0fc28 100644 --- a/app/repositories/report/ReportRepositoryInterface.scala +++ b/app/repositories/report/ReportRepositoryInterface.scala @@ -24,7 +24,6 @@ trait ReportRepositoryInterface extends CRUDRepositoryInterface[Report] { def streamReports: Source[Report, NotUsed] def streamAll: DatabasePublisher[((Report, Option[Company]), Option[BarcodeProduct])] - def cloudWord(companyId: UUID): Future[List[ReportWordOccurrence]] def findSimilarReportList( @@ -78,6 +77,7 @@ trait ReportRepositoryInterface extends CRUDRepositoryInterface[Report] { def getByStatusAndExpired(status: List[ReportStatus], now: OffsetDateTime): Future[List[Report]] + def getOldReports(createdBefore: OffsetDateTime): Future[List[Report]] def getPendingReports(companiesIds: List[UUID]): Future[List[Report]] def getPhoneReports(start: Option[LocalDate], end: Option[LocalDate]): Future[List[Report]] diff --git a/app/tasks/report/OldReportsRgpdDeletionTask.scala b/app/tasks/report/OldReportsRgpdDeletionTask.scala new file mode 100644 index 00000000..c0e951ca --- /dev/null +++ b/app/tasks/report/OldReportsRgpdDeletionTask.scala @@ -0,0 +1,51 @@ +package tasks.report + +import config.TaskConfiguration +import org.apache.pekko.actor.ActorSystem +import repositories.report.ReportRepositoryInterface +import repositories.tasklock.TaskRepositoryInterface +import tasks.ScheduledTask +import tasks.model.TaskSettings.DailyTaskSettings +import controllers.error.AppError.ServerError +import orchestrators.RgpdOrchestrator + +import java.time.OffsetDateTime +import scala.concurrent.ExecutionContext +import scala.concurrent.Future + +class OldReportsRgpdDeletionTask( + actorSystem: ActorSystem, + reportRepository: ReportRepositoryInterface, + rgpdOrchestrator: RgpdOrchestrator, + taskConfiguration: TaskConfiguration, + taskRepository: TaskRepositoryInterface +)(implicit val executionContext: ExecutionContext) + extends ScheduledTask(9, "old_reports_rgpd_deletion_task", taskRepository, actorSystem, taskConfiguration) { + + override val taskSettings = DailyTaskSettings(startTime = taskConfiguration.oldReportsRgpdDeletion.startTime) + + // The maximum number of reports created in single day so far is 2335 on 2023-05-23 + // So we shouldn't have to delete more than that each day + private val maxReportsAllowedToDelete = 2500 + + override def runTask(): Future[Unit] = { + val createdBefore = OffsetDateTime.now().minusYears(5) + for { + veryOldReports <- reportRepository.getOldReports(createdBefore) + _ = if (veryOldReports.length > maxReportsAllowedToDelete) { + // Safety : if we had some date-related bug in this task, it could empty our whole database! + throw ServerError( + s"Rgpd deletion failed, it was going to delete ${veryOldReports.length} reports!" + ) + } + _ <- veryOldReports.foldLeft(Future.successful(())) { case (previous, report) => + for { + _ <- previous + _ <- rgpdOrchestrator.deleteRGPD(report) + } yield () + } + + } yield () + } + +} diff --git a/conf/common/task.conf b/conf/common/task.conf index b89f0b34..15e0753e 100644 --- a/conf/common/task.conf +++ b/conf/common/task.conf @@ -58,6 +58,11 @@ task { start-time = ${?OLD_REPORT_EXPORT_DELETION_TASK_START_TIME} } + old-reports-rgpd-deletion { + start-time = "07:30:00" + start-time = ${?OLD_REPORTS_RGPD_DELETION_TASK_START_TIME} + } + sample-data { active = false active = ${?SAMPLE_DATA_GENERATION_TASK_ACTIVE} From 0ccf00cbb5a089c78d3e7e4742681243e8fb833c Mon Sep 17 00:00:00 2001 From: eletallbetagouv <107104509+eletallbetagouv@users.noreply.github.com> Date: Tue, 22 Oct 2024 16:38:11 +0300 Subject: [PATCH 2/4] Add some logs around the task --- app/orchestrators/RgpdOrchestrator.scala | 4 +++- app/tasks/report/OldReportsRgpdDeletionTask.scala | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/app/orchestrators/RgpdOrchestrator.scala b/app/orchestrators/RgpdOrchestrator.scala index 41afe006..07fd2ba6 100644 --- a/app/orchestrators/RgpdOrchestrator.scala +++ b/app/orchestrators/RgpdOrchestrator.scala @@ -20,13 +20,14 @@ class RgpdOrchestrator( engagementOrchestrator: EngagementOrchestrator, reportRepository: ReportRepositoryInterface, reportFileOrchestrator: ReportFileOrchestrator, - eventRepository: EventRepositoryInterface, + eventRepository: EventRepositoryInterface )(implicit val executionContext: ExecutionContext) { val logger = Logger(this.getClass) def deleteRGPD( report: Report ): Future[Report] = { + logger.info(s"Emptying report ${report.id} for RGPD") val emptiedReport = report.copy( firstName = "", lastName = "", @@ -51,6 +52,7 @@ class RgpdOrchestrator( _ <- reportConsumerReviewOrchestrator.deleteDetails(emptiedReport.id) _ <- engagementOrchestrator.deleteDetails(emptiedReport.id) _ <- reportFileOrchestrator.removeFromReportId(emptiedReport.id) + _ = logger.info(s"Report ${report.id} was emptied") } yield emptiedReport } diff --git a/app/tasks/report/OldReportsRgpdDeletionTask.scala b/app/tasks/report/OldReportsRgpdDeletionTask.scala index c0e951ca..10a5c91f 100644 --- a/app/tasks/report/OldReportsRgpdDeletionTask.scala +++ b/app/tasks/report/OldReportsRgpdDeletionTask.scala @@ -31,6 +31,7 @@ class OldReportsRgpdDeletionTask( override def runTask(): Future[Unit] = { val createdBefore = OffsetDateTime.now().minusYears(5) for { + // TODO ajouter filtre : qui ne sont pas dejà dans l'état RGPD veryOldReports <- reportRepository.getOldReports(createdBefore) _ = if (veryOldReports.length > maxReportsAllowedToDelete) { // Safety : if we had some date-related bug in this task, it could empty our whole database! @@ -38,6 +39,7 @@ class OldReportsRgpdDeletionTask( s"Rgpd deletion failed, it was going to delete ${veryOldReports.length} reports!" ) } + _ = logger.info(s"Found ${veryOldReports.length} that need to be emptied for RGPD (older than ${createdBefore})") _ <- veryOldReports.foldLeft(Future.successful(())) { case (previous, report) => for { _ <- previous From f5ca2b07665b340326bd73e7175e545ce70d81eb Mon Sep 17 00:00:00 2001 From: eletallbetagouv <107104509+eletallbetagouv@users.noreply.github.com> Date: Tue, 22 Oct 2024 16:43:51 +0300 Subject: [PATCH 3/4] do not process those already processed --- app/repositories/report/ReportRepository.scala | 4 +++- app/repositories/report/ReportRepositoryInterface.scala | 2 +- app/tasks/report/OldReportsRgpdDeletionTask.scala | 3 +-- test/controllers/report/ReportRepositoryMock.scala | 2 ++ 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/app/repositories/report/ReportRepository.scala b/app/repositories/report/ReportRepository.scala index 0a2344d7..7619c9b8 100644 --- a/app/repositories/report/ReportRepository.scala +++ b/app/repositories/report/ReportRepository.scala @@ -7,6 +7,7 @@ import models._ import models.barcode.BarcodeProduct import models.company.Company import models.report.ReportResponseType.ACCEPTED +import models.report.ReportStatus.SuppressionRGPD import models.report._ import models.report.reportmetadata.ReportMetadata import models.report.reportmetadata.ReportWithMetadata @@ -404,9 +405,10 @@ class ReportRepository(override val dbConfig: DatabaseConfig[JdbcProfile])(impli .result ) - def getOldReports(createdBefore: OffsetDateTime): Future[List[Report]] = db.run( + def getOldReportsNotRgpdDeleted(createdBefore: OffsetDateTime): Future[List[Report]] = db.run( table .filter(_.creationDate < createdBefore) + .filter(_.status =!= SuppressionRGPD.entryName) .to[List] .result ) diff --git a/app/repositories/report/ReportRepositoryInterface.scala b/app/repositories/report/ReportRepositoryInterface.scala index 9ef0fc28..9b4eff10 100644 --- a/app/repositories/report/ReportRepositoryInterface.scala +++ b/app/repositories/report/ReportRepositoryInterface.scala @@ -77,7 +77,7 @@ trait ReportRepositoryInterface extends CRUDRepositoryInterface[Report] { def getByStatusAndExpired(status: List[ReportStatus], now: OffsetDateTime): Future[List[Report]] - def getOldReports(createdBefore: OffsetDateTime): Future[List[Report]] + def getOldReportsNotRgpdDeleted(createdBefore: OffsetDateTime): Future[List[Report]] def getPendingReports(companiesIds: List[UUID]): Future[List[Report]] def getPhoneReports(start: Option[LocalDate], end: Option[LocalDate]): Future[List[Report]] diff --git a/app/tasks/report/OldReportsRgpdDeletionTask.scala b/app/tasks/report/OldReportsRgpdDeletionTask.scala index 10a5c91f..d663d844 100644 --- a/app/tasks/report/OldReportsRgpdDeletionTask.scala +++ b/app/tasks/report/OldReportsRgpdDeletionTask.scala @@ -31,8 +31,7 @@ class OldReportsRgpdDeletionTask( override def runTask(): Future[Unit] = { val createdBefore = OffsetDateTime.now().minusYears(5) for { - // TODO ajouter filtre : qui ne sont pas dejà dans l'état RGPD - veryOldReports <- reportRepository.getOldReports(createdBefore) + veryOldReports <- reportRepository.getOldReportsNotRgpdDeleted(createdBefore) _ = if (veryOldReports.length > maxReportsAllowedToDelete) { // Safety : if we had some date-related bug in this task, it could empty our whole database! throw ServerError( diff --git a/test/controllers/report/ReportRepositoryMock.scala b/test/controllers/report/ReportRepositoryMock.scala index a57f6617..a7b8e769 100644 --- a/test/controllers/report/ReportRepositoryMock.scala +++ b/test/controllers/report/ReportRepositoryMock.scala @@ -111,4 +111,6 @@ class ReportRepositoryMock(database: mutable.Map[UUID, Report] = mutable.Map.emp def streamAll: DatabasePublisher[((Report, Option[Company]), Option[BarcodeProduct])] = ??? override def streamReports: Source[Report, NotUsed] = ??? + + override def getOldReportsNotRgpdDeleted(createdBefore: OffsetDateTime): Future[List[Report]] = ??? } From 45b23a1aa9818b6fbff13cd8a828f5d26506b8cc Mon Sep 17 00:00:00 2001 From: eletallbetagouv <107104509+eletallbetagouv@users.noreply.github.com> Date: Tue, 22 Oct 2024 18:31:25 +0300 Subject: [PATCH 4/4] fix tests --- test/tasks/report/ReportRemindersTaskUnitSpec.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/test/tasks/report/ReportRemindersTaskUnitSpec.scala b/test/tasks/report/ReportRemindersTaskUnitSpec.scala index a66b7816..ca42a3c8 100644 --- a/test/tasks/report/ReportRemindersTaskUnitSpec.scala +++ b/test/tasks/report/ReportRemindersTaskUnitSpec.scala @@ -49,6 +49,7 @@ class ReportRemindersTaskUnitSpec extends Specification with FutureMatchers { reportClosure = null, orphanReportFileDeletion = null, oldReportExportDeletion = null, + oldReportsRgpdDeletion = null, reportReminders = ReportRemindersTaskConfiguration( startTime = LocalTime.of(2, 0), intervalInHours = 1.day,