[TRELLO-2608] Rework some request scanning the whole report table

app/controllers/ReportedPhoneController.scala

@@ -7,7 +7,7 @@ import org.apache.pekko.actor.typed
 import authentication.Authenticator
 import models._
 import play.api.Logger
-import play.api.libs.json.Json
+import play.api.libs.json.{JsObject, Json}
 import play.api.mvc.ControllerComponents
 import repositories.asyncfiles.AsyncFileRepositoryInterface
 import repositories.company.CompanyRepositoryInterface
@@ -31,29 +31,24 @@ class ReportedPhoneController(
   implicit val timeout: org.apache.pekko.util.Timeout = 5.seconds
   val logger: Logger                                  = Logger(this.getClass)
 
-  def fetchGrouped(q: Option[String], start: Option[String], end: Option[String]) =
+  def fetchGrouped(q: Option[String], start: Option[String], end: Option[String], offset: Option[Long], limit: Option[Int]) =
     SecuredAction.andThen(WithRole(UserRole.AdminsAndReadOnlyAndCCRF)).async { _ =>
       reportRepository
-        .getPhoneReports(DateUtils.parseDate(start), DateUtils.parseDate(end))
+        .getPhoneReports(q, DateUtils.parseDate(start), DateUtils.parseDate(end), offset, limit)
         .map(reports =>
           Ok(
             Json.toJson(
               reports
-                .groupBy(report => (report.phone, report.companySiret, report.companyName, report.category))
-                .collect {
-                  case ((Some(phone), siretOpt, companyNameOpt, category), reports) if q.forall(phone.contains(_)) =>
-                    ((phone, siretOpt, companyNameOpt, category), reports.length)
-                }
-                .map { case ((phone, siretOpt, companyNameOpt, category), count) =>
+                .mapEntities { case ((phone, siretOpt, companyNameOpt, category), count) =>
                   Json.obj(
-                    "phone"       -> phone,
-                    "siret"       -> siretOpt,
+                    "phone" -> phone,
+                    "siret" -> siretOpt,
                     "companyName" -> companyNameOpt,
-                    "category"    -> category,
-                    "count"       -> count
+                    "category" -> category,
+                    "count" -> count
                   )
                 }
-            )
+            )(PaginatedResult.paginatedResultWrites[JsObject])
           )
         )
     }

app/controllers/WebsiteController.scala

@@ -78,11 +78,11 @@ class WebsiteController(
       } yield Ok(resultAsJson)
     }
 
-  def fetchUnregisteredHost(host: Option[String], start: Option[String], end: Option[String]) =
+  def fetchUnregisteredHost(host: Option[String], start: Option[String], end: Option[String], offset: Option[Long], limit: Option[Int]) =
     SecuredAction.andThen(WithRole(UserRole.AdminsAndReadOnlyAndCCRF)).async { _ =>
       websitesOrchestrator
-        .fetchUnregisteredHost(host, start, end)
-        .map(websiteHostCount => Ok(Json.toJson(websiteHostCount)))
+        .fetchUnregisteredHost(host, start, end, offset, limit)
+        .map(websiteHostCount => Ok(Json.toJson(websiteHostCount)(paginatedResultWrites[WebsiteHostCount])))
     }
 
   def extractUnregisteredHost(q: Option[String], start: Option[String], end: Option[String]) =

app/models/report/ReportStatus.scala

@@ -49,7 +49,7 @@ object ReportStatus extends PlayEnum[ReportStatus] {
 
   implicit class ReportStatusOps(reportStatus: ReportStatus) {
     def isFinal: Boolean =
-      Seq(MalAttribue, ConsulteIgnore, NonConsulte, Infonde, PromesseAction, InformateurInterne, NA).contains(
+      Seq(MalAttribue, ConsulteIgnore, NonConsulte, Infonde, PromesseAction, InformateurInterne, NA, SuppressionRGPD).contains(
         reportStatus
       )
 

app/orchestrators/WebsitesOrchestrator.scala

@@ -248,11 +248,12 @@ class WebsitesOrchestrator(
   def fetchUnregisteredHost(
       host: Option[String],
       start: Option[String],
-      end: Option[String]
-  ): Future[List[WebsiteHostCount]] =
+      end: Option[String],
+      offset: Option[Long], limit: Option[Int]
+  ): Future[PaginatedResult[WebsiteHostCount]] =
     repository
-      .getUnkonwnReportCountByHost(host, DateUtils.parseDate(start), DateUtils.parseDate(end))
-      .map(_.map { case (host, count) => WebsiteHostCount(host, count) })
+      .getUnkonwnReportCountByHost(host, DateUtils.parseDate(start), DateUtils.parseDate(end), offset, limit)
+      .map(_.mapEntities { case (host, count) => WebsiteHostCount(host, count) })
 
   private def updatePreviousReportsAssociatedToWebsite(
       websiteHost: String,

app/repositories/report/ReportRepository.scala

@@ -10,7 +10,7 @@ import models.report.ReportResponseType.ACCEPTED
 import models.report._
 import models.report.reportmetadata.ReportMetadata
 import models.report.reportmetadata.ReportWithMetadata
-import repositories.CRUDRepository
+import repositories.{CRUDRepository, PaginateOps}
 import repositories.PostgresProfile.api._
 import repositories.barcode.BarcodeProductTable
 import repositories.company.CompanyTable
@@ -454,6 +454,27 @@ class ReportRepository(override val dbConfig: DatabaseConfig[JdbcProfile])(impli
           .result
       )
 
+  def getPhoneReports(q: Option[String], start: Option[LocalDate], end: Option[LocalDate], offset: Option[Long], limit: Option[Int]): Future[PaginatedResult[((Option[String], Option[SIRET], Option[String], String), Int)]] =
+    table
+      .filter(_.phone.isDefined)
+      .filterOpt(q) { case (table, p) =>
+        table.phone.like(s"%$p%")
+      }
+      .filterOpt(start) { case (table, start) =>
+        table.creationDate >= ZonedDateTime.of(start, LocalTime.MIN, ZoneOffset.UTC.normalized()).toOffsetDateTime
+      }
+      .filterOpt(end) { case (table, end) =>
+        table.creationDate < ZonedDateTime.of(end, LocalTime.MAX, ZoneOffset.UTC.normalized()).toOffsetDateTime
+      }
+      .groupBy(a => (a.phone, a.companySiret, a.companyName, a.category))
+      .map { case (a, b) => (a, b.length)}
+      .sortBy(_._2.desc)
+      .withPagination(db)(
+        maybeOffset = offset,
+        maybeLimit = limit,
+        maybePreliminaryAction = None
+      )
+
   override def getFor(userRole: Option[UserRole], id: UUID): Future[Option[ReportWithMetadata]] =
     for {
       maybeTuple <- db.run(

app/repositories/report/ReportRepositoryInterface.scala

@@ -11,6 +11,7 @@ import models.company.Company
 import models.report.reportmetadata.ReportWithMetadata
 import repositories.CRUDRepositoryInterface
 import slick.basic.DatabasePublisher
+import utils.SIRET
 
 import java.time.LocalDate
 import java.time.OffsetDateTime
@@ -82,6 +83,8 @@ trait ReportRepositoryInterface extends CRUDRepositoryInterface[Report] {
 
   def getPhoneReports(start: Option[LocalDate], end: Option[LocalDate]): Future[List[Report]]
 
+  def getPhoneReports(q: Option[String], start: Option[LocalDate], end: Option[LocalDate], offset: Option[Long], limit: Option[Int]): Future[PaginatedResult[((Option[String], Option[SIRET], Option[String], String), Int)]]
+
   def reportsCountBySubcategories(
       userRole: UserRole,
       filters: ReportsCountBySubcategoriesFilter,

app/repositories/website/WebsiteRepository.scala

@@ -9,7 +9,7 @@ import models.website.Website
 import models.website.WebsiteId
 import play.api.Logger
 import repositories.PostgresProfile.api._
-import repositories.TypedCRUDRepository
+import repositories.{PaginateOps, TypedCRUDRepository}
 import repositories.company.CompanyTable
 import repositories.report.ReportTable
 import repositories.website.WebsiteColumnType._
@@ -187,8 +187,8 @@ class WebsiteRepository(
 
   def getUnkonwnReportCountByHost(
       host: Option[String],
-      start: Option[LocalDate] = None,
-      end: Option[LocalDate] = None
+      start: Option[LocalDate],
+      end: Option[LocalDate],
   ): Future[List[(String, Int)]] = db
     .run(
       WebsiteTable.table
@@ -211,6 +211,34 @@ class WebsiteRepository(
         .result
     )
 
+  def getUnkonwnReportCountByHost(
+                                   host: Option[String],
+                                   start: Option[LocalDate],
+                                   end: Option[LocalDate],
+                                   offset: Option[Long], limit: Option[Int]
+                                 ): Future[PaginatedResult[(String, Int)]] =
+    WebsiteTable.table
+      .filter(t => host.fold(true.bind)(h => t.host like s"%${h}%"))
+      .filter(x => x.companyId.isEmpty && x.companyCountry.isEmpty)
+      .filterOpt(start) { case (table, start) =>
+        table.creationDate >= ZonedDateTime.of(start, LocalTime.MIN, ZoneOffset.UTC.normalized()).toOffsetDateTime
+      }
+      .filterOpt(end) { case (table, end) =>
+        table.creationDate < ZonedDateTime.of(end, LocalTime.MAX, ZoneOffset.UTC.normalized()).toOffsetDateTime
+      }
+      .joinLeft(ReportTable.table)
+      .on { (websiteTable, reportTable) =>
+        websiteTable.host === reportTable.host && reportTable.host.isDefined
+      }
+      .groupBy(_._1.host)
+      .map { case (host, report) => (host, report.map(_._2).size) }
+      .sortBy(_._2.desc)
+      .withPagination(db)(
+        maybeOffset = offset,
+        maybeLimit = limit,
+        maybePreliminaryAction = None
+      )
+
   def listNotAssociatedToCompany(host: String): Future[Seq[Website]] =
     db.run(
       table

app/repositories/website/WebsiteRepositoryInterface.scala

@@ -45,10 +45,17 @@ trait WebsiteRepositoryInterface extends TypedCRUDRepositoryInterface[Website, W
 
   def getUnkonwnReportCountByHost(
       host: Option[String],
-      start: Option[LocalDate] = None,
-      end: Option[LocalDate] = None
+      start: Option[LocalDate],
+      end: Option[LocalDate]
   ): Future[List[(String, Int)]]
 
+  def getUnkonwnReportCountByHost(
+                                   host: Option[String],
+                                   start: Option[LocalDate],
+                                   end: Option[LocalDate],
+                                   offset: Option[Long], limit: Option[Int]
+                                 ): Future[PaginatedResult[(String, Int)]]
+
   def listNotAssociatedToCompany(host: String): Future[Seq[Website]]
 
   def listIdentified(host: String): Future[Seq[Website]]

conf/routes

@@ -181,15 +181,15 @@ POST          /api/website-investigations                             controller
 GET           /api/websites                                           controllers.WebsiteController.fetchWithCompanies(host: Option[String], identificationStatus: Option[Seq[IdentificationStatus]], offset: Option[Long], limit: Option[Int],investigationStatus: Option[Seq[InvestigationStatus]],start: Option[OffsetDateTime],end: Option[OffsetDateTime],hasAssociation: Option[Boolean],isOpen: Option[Boolean])
 POST          /api/websites                                           controllers.WebsiteController.create()
 GET           /api/websites/search-url                                controllers.WebsiteController.searchByHost(url: String)
-GET           /api/websites/unregistered                              controllers.WebsiteController.fetchUnregisteredHost(q: Option[String], start: Option[String], end: Option[String])
+GET           /api/websites/unregistered                              controllers.WebsiteController.fetchUnregisteredHost(q: Option[String], start: Option[String], end: Option[String], offset: Option[Long], limit: Option[Int])
 GET           /api/websites/unregistered/extract                      controllers.WebsiteController.extractUnregisteredHost(q: Option[String], start: Option[String], end: Option[String])
 PUT           /api/websites/:id/company                               controllers.WebsiteController.updateCompany(id: WebsiteId)
 PUT           /api/websites/:id/country                               controllers.WebsiteController.updateCompanyCountry(id: WebsiteId, companyCountry : String)
 PUT           /api/websites/:id                                       controllers.WebsiteController.updateWebsiteIdentificationStatus(id: WebsiteId, identificationStatus :IdentificationStatus)
 DELETE        /api/websites/:id                                       controllers.WebsiteController.remove(id: WebsiteId)
 
 # Reported phones API
-GET           /api/reported-phones                                    controllers.ReportedPhoneController.fetchGrouped(q: Option[String], start: Option[String], end: Option[String])
+GET           /api/reported-phones                                    controllers.ReportedPhoneController.fetchGrouped(q: Option[String], start: Option[String], end: Option[String], offset: Option[Long], limit: Option[Int])
 GET           /api/reported-phones/extract                            controllers.ReportedPhoneController.extractPhonesGroupBySIRET(q: Option[String], start: Option[String], end: Option[String])
 
 # Report notifications block list API

test/controllers/ReportedPhoneControllerSpec.scala

@@ -61,7 +61,7 @@ The fetch phone group  SIRET endpoint should
                                                     """
 
   def e1 = {
-    val request = FakeRequest(GET, routes.ReportedPhoneController.fetchGrouped(None, None, None).toString)
+    val request = FakeRequest(GET, routes.ReportedPhoneController.fetchGrouped(None, None, None, None, None).toString)
       .withAuthCookie(adminUser.email, components.cookieAuthenticator)
     val result = route(app, request).get
     status(result) must beEqualTo(OK)

test/controllers/WebsiteControllerSpec.scala

@@ -92,7 +92,7 @@ The fetch unregistered host endpoint should
                                                     """
 
   def e1 = {
-    val request = FakeRequest(GET, routes.WebsiteController.fetchUnregisteredHost(None, None, None).toString)
+    val request = FakeRequest(GET, routes.WebsiteController.fetchUnregisteredHost(None, None, None, None, None).toString)
       .withAuthCookie(adminUser.email, components.cookieAuthenticator)
     val result = route(app, request).get
     status(result) must beEqualTo(OK)

test/controllers/report/ReportRepositoryMock.scala

@@ -11,7 +11,7 @@ import org.apache.pekko.NotUsed
 import org.apache.pekko.stream.scaladsl.Source
 import repositories.report.ReportRepositoryInterface
 import slick.basic.DatabasePublisher
-import utils.CRUDRepositoryMock
+import utils.{CRUDRepositoryMock, SIRET}
 
 import java.time.LocalDate
 import java.time.OffsetDateTime
@@ -85,6 +85,8 @@ class ReportRepositoryMock(database: mutable.Map[UUID, Report] = mutable.Map.emp
 
   override def getPhoneReports(start: Option[LocalDate], end: Option[LocalDate]): Future[List[Report]] = ???
 
+  override def getPhoneReports(q: Option[String], start: Option[LocalDate], end: Option[LocalDate], offset: Option[Long], limit: Option[Int]): Future[PaginatedResult[((Option[String], Option[SIRET], Option[String], String), Int)]] = ???
+
   override def cloudWord(companyId: UUID): Future[List[ReportWordOccurrence]] = ???
 
   override def reportsCountBySubcategories(