Improve speed of user list query

CHANGELOG.unreleased.md

@@ -18,14 +18,15 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 
 ### Changed
 - Improved loading speed of the annotation list. [#7410](https://github.com/scalableminds/webknossos/pull/7410)
+- Improved loading speed for the users list. [#7466](https://github.com/scalableminds/webknossos/pull/7466)
 - Admins and Team Managers can now also download job exports for jobs of other users, if they have the link. [#7462](https://github.com/scalableminds/webknossos/pull/7462)
 - Updated some dependencies of the backend code (play 2.9, sbt 1.9, minor upgrades for others) for optimized performance. [#7366](https://github.com/scalableminds/webknossos/pull/7366)
 - Processing jobs can now be distributed to multiple webknossos-workers with finer-grained configurability. Compare migration guide. [#7463](https://github.com/scalableminds/webknossos/pull/7463)
 - Removed Swagger/OpenAPI json description of the HTTP API. [#7494](https://github.com/scalableminds/webknossos/pull/7494)
 
 ### Fixed
 - Datasets with annotations can now be deleted. The concerning annotations can no longer be viewed but still be downloaded. [#7429](https://github.com/scalableminds/webknossos/pull/7429)
-- Fixed several deprecation warning for using antd's Tabs.TabPane components. [#7469](https://github.com/scalableminds/webknossos/pull/7469)
+- Fixed several deprecation warnings for using antd's Tabs.TabPane components. [#7469](https://github.com/scalableminds/webknossos/pull/7469)
 - Fixed problems when requests for loading data failed (could impact volume data consistency and rendering). [#7477](https://github.com/scalableminds/webknossos/pull/7477)
 - The settings page for non-wkw datasets no longer shows a wall of non-applying errors. [#7475](https://github.com/scalableminds/webknossos/pull/7475)
 - Fixed a bug where dataset deletion for ND datasets and datasets with coordinate transforms would not free the name even if no referencing annotations exist. [#7495](https://github.com/scalableminds/webknossos/pull/7495)

app/controllers/UserController.scala

@@ -238,11 +238,14 @@ class UserController @Inject()(userService: UserService,
       isAdmin: Option[Boolean]
   ): Action[AnyContent] = sil.SecuredAction.async { implicit request =>
     for {
-      users <- userDAO.findAllWithFilters(isEditable, isTeamManagerOrAdmin, isAdmin, request.identity)
-      js <- Fox.serialCombined(users.sortBy(_.lastName.toLowerCase))(u => userService.publicWrites(u, request.identity))
-    } yield {
-      Ok(Json.toJson(js))
-    }
+      (users, userCompactInfos) <- userDAO.findAllCompactWithFilters(isEditable,
+                                                                     isTeamManagerOrAdmin,
+                                                                     isAdmin,
+                                                                     request.identity)
+      zipped = users.zip(userCompactInfos)
+      js <- Fox.serialCombined(zipped.sortBy(_._1.lastName.toLowerCase))(u =>
+        userService.publicWritesCompact(u._1, request.identity, u._2))
+    } yield Ok(Json.toJson(js))
   }
 
   private val userUpdateReader =

app/models/user/User.scala

@@ -12,12 +12,14 @@ import com.scalableminds.webknossos.schema.Tables._
 import javax.inject.Inject
 import models.team._
 import play.api.libs.json._
+import slick.jdbc.GetResult
 import slick.jdbc.PostgresProfile.api._
 import slick.jdbc.TransactionIsolation.Serializable
 import slick.lifted.Rep
 import utils.sql.{SQLDAO, SimpleSQLDAO, SqlClient, SqlToken}
 import utils.ObjectId
 
+import java.sql.Timestamp
 import scala.concurrent.ExecutionContext
 
 object User {
@@ -60,6 +62,56 @@ case class User(
 
 }
 
+case class UserCompactInfo(
+    _id: String,
+    _multiUserId: String,
+    email: String,
+    firstname: String,
+    lastname: String,
+    userConfiguration: String,
+    isAdmin: Boolean,
+    isOrganizationOwner: Boolean,
+    isDatasetManager: Boolean,
+    isDeactivated: Boolean,
+    team_ids: String,
+    team_names: String,
+    team_managers: String,
+    experienceValues: String,
+    experienceDomains: String,
+    lastActivity: Timestamp,
+    organization_id: String,
+    organization_name: String,
+    novelUserExperienceInfos: String,
+    selectedTheme: String,
+    created: Timestamp,
+    lastTaskTypeId: Option[String],
+    isSuperUser: Boolean,
+    isEmailVerified: Boolean
+) {
+  def toUser(implicit ec: ExecutionContext): Fox[User] =
+    for {
+      userConfiguration <- Fox.box2Fox(parseAndValidateJson[JsObject](userConfiguration))
+    } yield {
+      User(
+        ObjectId(_id),
+        ObjectId(_multiUserId),
+        ObjectId(organization_id),
+        firstname,
+        lastname,
+        Instant.fromSql(lastActivity),
+        userConfiguration,
+        LoginInfo(User.default_login_provider_id, _id),
+        isAdmin,
+        isOrganizationOwner,
+        isDatasetManager,
+        isDeactivated,
+        isUnlisted = false,
+        Instant.fromSql(created),
+        lastTaskTypeId.map(ObjectId(_))
+      )
+    }
+}
+
 class UserDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
     extends SQLDAO[User, UsersRow, Users](sqlClient) {
   protected val collection = Users
@@ -92,14 +144,18 @@ class UserDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
     }
 
   override protected def readAccessQ(requestingUserId: ObjectId) =
-    q"""(_id in (select _user from webknossos.user_team_roles where _team in (select _team from webknossos.user_team_roles where _user = $requestingUserId and isTeamManager)))
-        or (_organization in (select _organization from webknossos.users_ where _id = $requestingUserId and isAdmin))
-        or _id = $requestingUserId"""
+    readAccessQWithPrefix(requestingUserId, SqlToken.raw(""))
+
+  protected def readAccessQWithPrefix(requestingUserId: ObjectId, userPrefix: SqlToken) =
+    q"""(${userPrefix}_id in (select _user from webknossos.user_team_roles where _team in (select _team from webknossos.user_team_roles where _user = $requestingUserId and isTeamManager)))
+        or (${userPrefix}_organization in (select _organization from webknossos.users_ where _id = $requestingUserId and isAdmin))
+        or ${userPrefix}_id = $requestingUserId"""
   override protected def deleteAccessQ(requestingUserId: ObjectId) =
     q"_organization in (select _organization from webknossos.users_ where _id = $requestingUserId and isAdmin)"
 
-  private def listAccessQ(requestingUserId: ObjectId) =
-    q"""(${readAccessQ(requestingUserId)})
+  private def listAccessQ(requestingUserId: ObjectId) = listAccessQWithPrefix(requestingUserId, SqlToken.raw(""))
+  private def listAccessQWithPrefix(requestingUserId: ObjectId, prefix: SqlToken) =
+    q"""(${readAccessQWithPrefix(requestingUserId, prefix)})
         and
         (
           isUnlisted = false
@@ -130,48 +186,100 @@ class UserDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
   def buildSelectionPredicates(isEditableOpt: Option[Boolean],
                                isTeamManagerOrAdminOpt: Option[Boolean],
                                isAdminOpt: Option[Boolean],
-                               requestingUser: User)(implicit ctx: DBAccessContext): Fox[SqlToken] =
+                               requestingUser: User,
+                               userPrefix: SqlToken)(implicit ctx: DBAccessContext): Fox[SqlToken] =
     for {
-      accessQuery <- accessQueryFromAccessQ(listAccessQ)
+      accessQuery <- accessQueryFromAccessQWithPrefix(listAccessQWithPrefix, userPrefix)
       editablePredicate = isEditableOpt match {
         case Some(isEditable) =>
           val usersInTeamsManagedByRequestingUser =
             q"(SELECT _user FROM webknossos.user_team_roles WHERE _team IN (SELECT _team FROM webknossos.user_team_roles WHERE _user = ${requestingUser._id}  AND isTeamManager)))"
           if (isEditable) {
-            q"(_id IN $usersInTeamsManagedByRequestingUser OR (${requestingUser.isAdmin} AND _organization = ${requestingUser._organization})"
+            q"(${userPrefix}_id IN $usersInTeamsManagedByRequestingUser OR (${requestingUser.isAdmin} AND ${userPrefix}_organization = ${requestingUser._organization})"
           } else {
-            q"(_id NOT IN $usersInTeamsManagedByRequestingUser AND (NOT (${requestingUser.isAdmin} AND _organization = ${requestingUser._organization}))"
+            q"(${userPrefix}_id NOT IN $usersInTeamsManagedByRequestingUser AND (NOT (${requestingUser.isAdmin} AND ${userPrefix}_organization = ${requestingUser._organization}))"
           }
         case None => q"${true}"
       }
       isTeamManagerOrAdminPredicate = isTeamManagerOrAdminOpt match {
         case Some(isTeamManagerOrAdmin) =>
           val teamManagers = q"(SELECT _user FROM webknossos.user_team_roles WHERE isTeamManager)"
           if (isTeamManagerOrAdmin) {
-            q"_id IN $teamManagers OR isAdmin"
+            q"${userPrefix}_id IN $teamManagers OR ${userPrefix}isAdmin"
           } else {
-            q"_id NOT IN $teamManagers AND NOT isAdmin"
+            q"${userPrefix}_id NOT IN $teamManagers AND NOT ${userPrefix}isAdmin"
           }
         case None => q"${true}"
       }
-      adminPredicate = isAdminOpt.map(isAdmin => q"isAdmin = $isAdmin").getOrElse(q"${true}")
+      adminPredicate = isAdminOpt.map(isAdmin => q"${userPrefix}isAdmin = $isAdmin").getOrElse(q"${true}")
     } yield q"""
         ($editablePredicate) AND
         ($isTeamManagerOrAdminPredicate) AND
         ($adminPredicate) AND
         $accessQuery
        """
 
-  def findAllWithFilters(isEditable: Option[Boolean],
-                         isTeamManagerOrAdmin: Option[Boolean],
-                         isAdmin: Option[Boolean],
-                         requestingUser: User)(implicit ctx: DBAccessContext): Fox[List[User]] =
+  implicit def GetResultUserCompactInfo(implicit e0: GetResult[String],
+                                        e1: GetResult[java.sql.Timestamp],
+                                        e2: GetResult[Boolean],
+                                        e3: GetResult[Option[String]]): GetResult[UserCompactInfo] = GetResult { prs =>
+    import prs._
+    // format: off
+    UserCompactInfo(<<[String],<<[String],<<[String],<<[String],<<[String],<<[String],<<[Boolean],<<[Boolean],
+      <<[Boolean],<<[Boolean],<<[String],<<[String],<<[String],<<[String], <<[String],<<[java.sql.Timestamp],<<[String],
+      <<[String],<<[String],<<[String],<<[java.sql.Timestamp],<<?[String],<<[Boolean],<<[Boolean]
+    )
+    // format: on
+  }
+  def findAllCompactWithFilters(
+      isEditable: Option[Boolean],
+      isTeamManagerOrAdmin: Option[Boolean],
+      isAdmin: Option[Boolean],
+      requestingUser: User)(implicit ctx: DBAccessContext): Fox[(List[User], List[UserCompactInfo])] =
     for {
-
-      selectionPredicates <- buildSelectionPredicates(isEditable, isTeamManagerOrAdmin, isAdmin, requestingUser)
-      r <- run(q"select $columns from $existingCollectionName where $selectionPredicates".as[UsersRow])
-      parsed <- parseAll(r)
-    } yield parsed
+      selectionPredicates <- buildSelectionPredicates(isEditable,
+                                                      isTeamManagerOrAdmin,
+                                                      isAdmin,
+                                                      requestingUser,
+                                                      SqlToken.raw("u."))
+      r <- run(q"""
+          SELECT
+            u._id,
+            m._id,
+            m.email,
+            u.firstname,
+            u.lastname,
+            u.userConfiguration,
+            u.isadmin,
+            u.isorganizationowner,
+            u.isdatasetmanager,
+            u.isdeactivated,
+            STRING_AGG(t._id, ',') AS team_ids,
+            STRING_AGG(t.name, ',') AS team_names,
+            STRING_AGG(utr.isteammanager :: TEXT, ',') AS team_managers,
+            STRING_AGG(ux.value :: TEXT, ',') AS experience_values,
+            STRING_AGG(ux.domain, ',') AS experience_domains,
+            u.lastactivity,
+            o._id,
+            o.name,
+            m.noveluserexperienceinfos,
+            m.selectedtheme,
+            u.created,
+            u.lasttasktypeid,
+            m.issuperuser,
+            m.isemailverified
+        FROM webknossos.users AS u
+        INNER JOIN webknossos.organizations o on o._id = u._organization
+        INNER JOIN webknossos.multiusers m on u._multiuser = m._id
+        INNER JOIN webknossos.user_team_roles utr on utr._user = u._id
+        INNER JOIN webknossos.teams t on t._id = utr._team
+        LEFT JOIN webknossos.user_experiences ux on ux._user = u._id
+        WHERE $selectionPredicates
+        GROUP BY u._id, o._id, m._id, m.email, m.noveluserexperienceinfos, m.selectedtheme, m.issuperuser, m.isemailverified
+         """.as[UserCompactInfo])
+      users <- Fox.combined(r.toList.map(_.toUser))
+      compactInfos = r.toList
+    } yield (users, compactInfos)
 
   def findAllByTeams(teamIds: List[ObjectId])(implicit ctx: DBAccessContext): Fox[List[User]] =
     if (teamIds.isEmpty) Fox.successful(List())

app/models/user/UserService.scala

@@ -368,6 +368,47 @@ class UserService @Inject()(conf: WkConf,
     }
   }
 
+  def publicWritesCompact(user: User, requestingUser: User, userCompactInfo: UserCompactInfo): Fox[JsObject] =
+    for {
+      isEditable <- isEditableBy(user, requestingUser)
+      splitString = (s: String) => Option(s).map(str => str.split(",")).getOrElse(Array[String]()).toSeq
+      teamsJson = splitString(userCompactInfo.team_ids).indices.map(
+        idx =>
+          Json.obj(
+            "id" -> splitString(userCompactInfo.team_ids)(idx),
+            "name" -> splitString(userCompactInfo.team_names)(idx),
+            "isTeamManager" -> splitString(userCompactInfo.team_managers)(idx).toBoolean
+        ))
+      experienceJson = Json.obj(
+        splitString(userCompactInfo.experienceValues).indices.map(idx =>
+          (splitString(userCompactInfo.experienceDomains)(idx),
+           Json.toJsFieldJsValueWrapper(splitString(userCompactInfo.experienceValues)(idx).toInt))): _*)
+      novelUserExperienceInfos <- Json.parse(userCompactInfo.novelUserExperienceInfos).validate[JsObject]
+    } yield {
+      Json.obj(
+        "id" -> user._id.toString,
+        "email" -> userCompactInfo.email,
+        "firstName" -> user.firstName,
+        "lastName" -> user.lastName,
+        "isAdmin" -> user.isAdmin,
+        "isOrganizationOwner" -> user.isOrganizationOwner,
+        "isDatasetManager" -> user.isDatasetManager,
+        "isActive" -> !user.isDeactivated,
+        "teams" -> teamsJson,
+        "experiences" -> experienceJson,
+        "lastActivity" -> user.lastActivity,
+        "isAnonymous" -> false,
+        "isEditable" -> isEditable,
+        "organization" -> userCompactInfo.organization_name,
+        "novelUserExperienceInfos" -> novelUserExperienceInfos,
+        "selectedTheme" -> userCompactInfo.selectedTheme,
+        "created" -> user.created,
+        "lastTaskTypeId" -> user.lastTaskTypeId.map(_.toString),
+        "isSuperUser" -> userCompactInfo.isSuperUser,
+        "isEmailVerified" -> userCompactInfo.isEmailVerified,
+      )
+    }
+
   def compactWrites(user: User): Fox[JsObject] = {
     implicit val ctx: DBAccessContext = GlobalAccessContext
     for {