From a955217ad40ef7c63a00269122e8ac646dfed403 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 18 Jun 2024 15:59:46 +0200 Subject: [PATCH 001/150] WIP: Unified Annotation Versioning --- webknossos-datastore/proto/Annotation.proto | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 webknossos-datastore/proto/Annotation.proto diff --git a/webknossos-datastore/proto/Annotation.proto b/webknossos-datastore/proto/Annotation.proto new file mode 100644 index 00000000000..74742e3cddd --- /dev/null +++ b/webknossos-datastore/proto/Annotation.proto @@ -0,0 +1,16 @@ +syntax = "proto2"; + +package com.scalableminds.webknossos.datastore; + +message AnnotationProto { + optional string name = 1; + optional string description = 2; + required int64 version = 3; + repeated AnnotationLayerProto layers = 4; +} + +message AnnotationLayerProto { + required string name = 1; + required int64 version = 2; + required string tracingId = 3; +} From a4361fe0ea337457c61100314e883227ae590a4d Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 19 Jun 2024 16:59:19 +0200 Subject: [PATCH 002/150] initialize route --- fossildb/run.sh | 2 +- .../controllers/DSAnnotationController.scala | 40 +++++++++++++++++++ .../tracings/TracingDataStore.scala | 2 + ...alableminds.webknossos.tracingstore.routes | 2 + 4 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala diff --git a/fossildb/run.sh b/fossildb/run.sh index 55853d3b302..39a622d2ee8 100755 --- a/fossildb/run.sh +++ b/fossildb/run.sh @@ -15,6 +15,6 @@ if [ ! -f "$JAR" ] || [ ! "$CURRENT_VERSION" == "$VERSION" ]; then fi # Note that the editableMappings column is no longer used by wk. Still here for backwards compatibility. -COLLECTIONS="skeletons,skeletonUpdates,volumes,volumeData,volumeUpdates,volumeSegmentIndex,editableMappings,editableMappingUpdates,editableMappingsInfo,editableMappingsAgglomerateToGraph,editableMappingsSegmentToAgglomerate" +COLLECTIONS="skeletons,skeletonUpdates,volumes,volumeData,volumeUpdates,volumeSegmentIndex,editableMappings,editableMappingUpdates,editableMappingsInfo,editableMappingsAgglomerateToGraph,editableMappingsSegmentToAgglomerate,annotations,annotationUpdates" exec java -jar "$JAR" -c "$COLLECTIONS" -d "$FOSSILDB_HOME/data" -b "$FOSSILDB_HOME/backup" diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala new file mode 100644 index 00000000000..a5fc4b2355c --- /dev/null +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala @@ -0,0 +1,40 @@ +package com.scalableminds.webknossos.tracingstore.controllers + +import com.google.inject.Inject +import com.scalableminds.webknossos.datastore.Annotation.AnnotationProto +import com.scalableminds.webknossos.datastore.controllers.Controller +import com.scalableminds.webknossos.datastore.services.{AccessTokenService, UserAccessRequest} +import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} +import play.api.mvc.{Action, AnyContent, PlayBodyParsers} + +import scala.concurrent.{ExecutionContext, Future} + +class DSAnnotationController @Inject()(accessTokenService: AccessTokenService, tracingDataStore: TracingDataStore)( + implicit ec: ExecutionContext, + bodyParsers: PlayBodyParsers) + extends Controller + with KeyValueStoreImplicits { + + def initialize(annotationId: String, token: Option[String]): Action[AnyContent] = + Action.async { implicit request => + log() { + accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { + for { + _ <- tracingDataStore.annotations.put(annotationId, 0L, AnnotationProto(version = 0L)) + } yield Ok + } + } + } +} + +// get version history + +// update layer + +// restore of layer + +// delete layer + +// add layer + +// skeleton + volume routes can now take annotationVersion diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala index 37df9decf09..629d79d18a9 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala @@ -42,6 +42,8 @@ class TracingDataStore @Inject()(config: TracingStoreConfig, lazy val editableMappingUpdates = new FossilDBClient("editableMappingUpdates", config, slackNotificationService) + lazy val annotations = new FossilDBClient("annotations", config, slackNotificationService) + private def shutdown(): Unit = { healthClient.shutdown() skeletons.shutdown() diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 7598dd0baa6..c876fc180e1 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -5,6 +5,8 @@ # Health endpoint GET /health @com.scalableminds.webknossos.tracingstore.controllers.Application.health +POST /annotation/initialize @com.scalableminds.webknossos.tracingstore.controllers.DSAnnotationController.initialize(annotationId: String, token: Option[String]) + # Volume tracings POST /volume/save @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.save(token: Option[String]) POST /volume/:tracingId/initialData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.initialData(token: Option[String], tracingId: String, minResolution: Option[Int], maxResolution: Option[Int]) From 5a60d17049416fa9184c98944f044edc1f1d6a89 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 20 Jun 2024 14:08:07 +0200 Subject: [PATCH 003/150] update actions --- webknossos-datastore/proto/Annotation.proto | 41 ++++++++++++++- .../annotation/DSAnnotationService.scala | 51 +++++++++++++++++++ .../controllers/DSAnnotationController.scala | 2 + 3 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala diff --git a/webknossos-datastore/proto/Annotation.proto b/webknossos-datastore/proto/Annotation.proto index 74742e3cddd..77e9e902a40 100644 --- a/webknossos-datastore/proto/Annotation.proto +++ b/webknossos-datastore/proto/Annotation.proto @@ -2,6 +2,11 @@ syntax = "proto2"; package com.scalableminds.webknossos.datastore; +enum AnnotationLayerTypeProto { + skeleton = 1; + volume = 2; +} + message AnnotationProto { optional string name = 1; optional string description = 2; @@ -10,7 +15,39 @@ message AnnotationProto { } message AnnotationLayerProto { + required string tracingId = 1; + required string name = 2; + required int64 version = 3; + optional int64 editableMappingVersion = 4; + required AnnotationLayerTypeProto type = 5; +} + +message AddLayerAnnotationUpdateAction { required string name = 1; - required int64 version = 2; - required string tracingId = 3; + required string tracingId = 2; + required AnnotationLayerTypeProto type = 5; +} + +message DeleteLayerAnnotationUpdateAction { + required string tracingId = 1; +} + +message UpdateLayerAnnotationUpdateAction { + required string tracingId = 1; + required int64 layerVersion = 2; +} + +message UpdateLayerEditableMappingAnnotationUpdateAction { + required string tracingId = 1; + required int64 editableMappingVersion = 2; +} + +message UpdateLayerMetadataAnnotationUpdateAction { + required string tracingId = 1; + required string name = 2; +} + +message UpdateMetadataAnnotationUpdateAction { + optional string name = 1; + optional string description = 2; } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala new file mode 100644 index 00000000000..29b408c4ef8 --- /dev/null +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala @@ -0,0 +1,51 @@ +package com.scalableminds.webknossos.tracingstore.annotation + +import com.scalableminds.util.tools.Fox +import com.scalableminds.webknossos.datastore.Annotation.{ + AddLayerAnnotationUpdateAction, + AnnotationLayerProto, + AnnotationProto, + DeleteLayerAnnotationUpdateAction, + UpdateLayerAnnotationUpdateAction, + UpdateLayerEditableMappingAnnotationUpdateAction, + UpdateLayerMetadataAnnotationUpdateAction, + UpdateMetadataAnnotationUpdateAction +} +import scalapb.GeneratedMessage + +import javax.inject.Inject +import scala.concurrent.ExecutionContext + +class DSAnnotationService @Inject()() { + def storeUpdate(updateAction: GeneratedMessage)(implicit ec: ExecutionContext): Fox[Unit] = Fox.successful(()) + + def applyUpdate(annotation: AnnotationProto, updateAction: GeneratedMessage): AnnotationProto = { + + val withAppliedChange = updateAction match { + case a: AddLayerAnnotationUpdateAction => + annotation.copy( + layers = annotation.layers :+ AnnotationLayerProto(a.tracingId, + a.name, + version = 0L, + editableMappingVersion = None, + `type` = a.`type`)) + case a: DeleteLayerAnnotationUpdateAction => + annotation.copy(layers = annotation.layers.filter(_.tracingId != a.tracingId)) + case a: UpdateLayerAnnotationUpdateAction => + annotation.copy( + layers = annotation.layers.map(l => if (l.tracingId == a.tracingId) l.copy(version = a.layerVersion) else l)) + case a: UpdateLayerEditableMappingAnnotationUpdateAction => + annotation.copy(layers = annotation.layers.map(l => + if (l.tracingId == a.tracingId) l.copy(editableMappingVersion = Some(a.editableMappingVersion)) else l)) + case a: UpdateLayerMetadataAnnotationUpdateAction => + annotation.copy( + layers = annotation.layers.map(l => if (l.tracingId == a.tracingId) l.copy(name = a.name) else l)) + case a: UpdateMetadataAnnotationUpdateAction => + annotation.copy(name = a.name, description = a.description) + // TODO error case + + } + withAppliedChange.copy(version = withAppliedChange.version + 1L) + } + +} diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala index a5fc4b2355c..31fbcc6e408 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala @@ -38,3 +38,5 @@ class DSAnnotationController @Inject()(accessTokenService: AccessTokenService, t // add layer // skeleton + volume routes can now take annotationVersion + +// Is an editable mapping a layer? From a698f11e6ba5c593cc830ddb995801907c336f48 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 20 Jun 2024 16:45:59 +0200 Subject: [PATCH 004/150] error case --- webknossos-datastore/proto/Annotation.proto | 2 + .../annotation/DSAnnotationService.scala | 55 ++++++++++--------- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/webknossos-datastore/proto/Annotation.proto b/webknossos-datastore/proto/Annotation.proto index 77e9e902a40..16c38629d66 100644 --- a/webknossos-datastore/proto/Annotation.proto +++ b/webknossos-datastore/proto/Annotation.proto @@ -51,3 +51,5 @@ message UpdateMetadataAnnotationUpdateAction { optional string name = 1; optional string description = 2; } + +// TODO restoreLayer? diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala index 29b408c4ef8..ae60f6c65c4 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala @@ -19,33 +19,36 @@ import scala.concurrent.ExecutionContext class DSAnnotationService @Inject()() { def storeUpdate(updateAction: GeneratedMessage)(implicit ec: ExecutionContext): Fox[Unit] = Fox.successful(()) - def applyUpdate(annotation: AnnotationProto, updateAction: GeneratedMessage): AnnotationProto = { + def newestMaterializableVersion(annotationId: String): Fox[Long] = ??? - val withAppliedChange = updateAction match { - case a: AddLayerAnnotationUpdateAction => - annotation.copy( - layers = annotation.layers :+ AnnotationLayerProto(a.tracingId, - a.name, - version = 0L, - editableMappingVersion = None, - `type` = a.`type`)) - case a: DeleteLayerAnnotationUpdateAction => - annotation.copy(layers = annotation.layers.filter(_.tracingId != a.tracingId)) - case a: UpdateLayerAnnotationUpdateAction => - annotation.copy( - layers = annotation.layers.map(l => if (l.tracingId == a.tracingId) l.copy(version = a.layerVersion) else l)) - case a: UpdateLayerEditableMappingAnnotationUpdateAction => - annotation.copy(layers = annotation.layers.map(l => - if (l.tracingId == a.tracingId) l.copy(editableMappingVersion = Some(a.editableMappingVersion)) else l)) - case a: UpdateLayerMetadataAnnotationUpdateAction => - annotation.copy( - layers = annotation.layers.map(l => if (l.tracingId == a.tracingId) l.copy(name = a.name) else l)) - case a: UpdateMetadataAnnotationUpdateAction => - annotation.copy(name = a.name, description = a.description) - // TODO error case + def applyUpdate(annotation: AnnotationProto, updateAction: GeneratedMessage)( + implicit ec: ExecutionContext): Fox[AnnotationProto] = + for { - } - withAppliedChange.copy(version = withAppliedChange.version + 1L) - } + withAppliedChange <- updateAction match { + case a: AddLayerAnnotationUpdateAction => + Fox.successful( + annotation.copy( + layers = annotation.layers :+ AnnotationLayerProto(a.tracingId, + a.name, + version = 0L, + editableMappingVersion = None, + `type` = a.`type`))) + case a: DeleteLayerAnnotationUpdateAction => + Fox.successful(annotation.copy(layers = annotation.layers.filter(_.tracingId != a.tracingId))) + case a: UpdateLayerAnnotationUpdateAction => + Fox.successful(annotation.copy(layers = annotation.layers.map(l => + if (l.tracingId == a.tracingId) l.copy(version = a.layerVersion) else l))) + case a: UpdateLayerEditableMappingAnnotationUpdateAction => + Fox.successful(annotation.copy(layers = annotation.layers.map(l => + if (l.tracingId == a.tracingId) l.copy(editableMappingVersion = Some(a.editableMappingVersion)) else l))) + case a: UpdateLayerMetadataAnnotationUpdateAction => + Fox.successful(annotation.copy(layers = annotation.layers.map(l => + if (l.tracingId == a.tracingId) l.copy(name = a.name) else l))) + case a: UpdateMetadataAnnotationUpdateAction => + Fox.successful(annotation.copy(name = a.name, description = a.description)) + case _ => Fox.failure("Received unsupported AnnotationUpdaetAction action") + } + } yield withAppliedChange.copy(version = withAppliedChange.version + 1L) } From 9113ee1f75c7cf6e5eb00b938c0d816dc387d468 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 20 Jun 2024 17:47:55 +0200 Subject: [PATCH 005/150] fix injector --- app/controllers/Application.scala | 15 +++++++++++++++ conf/webknossos.latest.routes | 1 + .../controllers/DSAnnotationController.scala | 9 +++++---- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/app/controllers/Application.scala b/app/controllers/Application.scala index 7d0ba191d18..18af7eaaef8 100755 --- a/app/controllers/Application.scala +++ b/app/controllers/Application.scala @@ -15,6 +15,11 @@ import utils.{ApiVersioning, StoreModules, WkConf} import javax.inject.Inject import scala.concurrent.ExecutionContext +import scalapb.GeneratedMessage +import com.scalableminds.webknossos.datastore.Annotation.{ + UpdateLayerAnnotationUpdateAction, + UpdateLayerMetadataAnnotationUpdateAction +} class Application @Inject()(actorSystem: ActorSystem, userService: UserService, @@ -30,6 +35,16 @@ class Application @Inject()(actorSystem: ActorSystem, private lazy val Mailer = actorSystem.actorSelection("/user/mailActor") + def test: Action[AnyContent] = Action.async { implicit request => + Fox.successful(Ok(matchThing(UpdateLayerMetadataAnnotationUpdateAction("id", "name")))) + } + + private def matchThing(m: GeneratedMessage): String = m match { + case u: UpdateLayerAnnotationUpdateAction => "update!" + case u: UpdateLayerMetadataAnnotationUpdateAction => "other update!" + case _ => "anything else!" + } + // Note: This route is used by external applications, keep stable def buildInfo: Action[AnyContent] = sil.UserAwareAction.async { for { diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes index b79feba1742..2f47c9d57a1 100644 --- a/conf/webknossos.latest.routes +++ b/conf/webknossos.latest.routes @@ -1,6 +1,7 @@ # Routes # This file defines all application routes (Higher priority routes first) # ~~~~ +GET /test controllers.Application.test() GET /buildinfo controllers.Application.buildInfo() GET /features controllers.Application.features() GET /health controllers.Application.health() diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala index 31fbcc6e408..e534925f6c0 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala @@ -3,15 +3,16 @@ package com.scalableminds.webknossos.tracingstore.controllers import com.google.inject.Inject import com.scalableminds.webknossos.datastore.Annotation.AnnotationProto import com.scalableminds.webknossos.datastore.controllers.Controller -import com.scalableminds.webknossos.datastore.services.{AccessTokenService, UserAccessRequest} +import com.scalableminds.webknossos.datastore.services.UserAccessRequest import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} +import com.scalableminds.webknossos.tracingstore.TracingStoreAccessTokenService import play.api.mvc.{Action, AnyContent, PlayBodyParsers} import scala.concurrent.{ExecutionContext, Future} -class DSAnnotationController @Inject()(accessTokenService: AccessTokenService, tracingDataStore: TracingDataStore)( - implicit ec: ExecutionContext, - bodyParsers: PlayBodyParsers) +class DSAnnotationController @Inject()( + accessTokenService: TracingStoreAccessTokenService, + tracingDataStore: TracingDataStore)(implicit ec: ExecutionContext, bodyParsers: PlayBodyParsers) extends Controller with KeyValueStoreImplicits { From 332eb19001705e53e696ebf4627d11def7169b89 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 8 Jul 2024 11:48:44 +0200 Subject: [PATCH 006/150] sparse versions --- webknossos-datastore/proto/Annotation.proto | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/webknossos-datastore/proto/Annotation.proto b/webknossos-datastore/proto/Annotation.proto index 16c38629d66..4fe56262b5c 100644 --- a/webknossos-datastore/proto/Annotation.proto +++ b/webknossos-datastore/proto/Annotation.proto @@ -17,9 +17,7 @@ message AnnotationProto { message AnnotationLayerProto { required string tracingId = 1; required string name = 2; - required int64 version = 3; - optional int64 editableMappingVersion = 4; - required AnnotationLayerTypeProto type = 5; + required AnnotationLayerTypeProto type = 4; } message AddLayerAnnotationUpdateAction { @@ -32,16 +30,6 @@ message DeleteLayerAnnotationUpdateAction { required string tracingId = 1; } -message UpdateLayerAnnotationUpdateAction { - required string tracingId = 1; - required int64 layerVersion = 2; -} - -message UpdateLayerEditableMappingAnnotationUpdateAction { - required string tracingId = 1; - required int64 editableMappingVersion = 2; -} - message UpdateLayerMetadataAnnotationUpdateAction { required string tracingId = 1; required string name = 2; From b46af6360fdcaecf44d1a6d83fe31883bb57ac33 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 8 Jul 2024 11:59:40 +0200 Subject: [PATCH 007/150] WIP: revert actions for editable mappings --- .../editablemapping/EditableMappingUpdateActions.scala | 5 +++++ .../tracings/editablemapping/EditableMappingUpdater.scala | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala index 652f13c96d9..0bde8f4473b 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala @@ -43,6 +43,11 @@ object MergeAgglomerateUpdateAction { implicit val jsonFormat: OFormat[MergeAgglomerateUpdateAction] = Json.format[MergeAgglomerateUpdateAction] } +case class RevertToVersionUpdateAction(sourceVersion: Long, actionTimestamp: Option[Long] = None) + extends EditableMappingUpdateAction { + override def addTimestamp(timestamp: Long): EditableMappingUpdateAction = this.copy(actionTimestamp = Some(timestamp)) +} + object EditableMappingUpdateAction { implicit object editableMappingUpdateActionFormat extends Format[EditableMappingUpdateAction] { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index 545c94c92cb..80a3f0151cb 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -103,6 +103,8 @@ class EditableMappingUpdater( applySplitAction(mapping, splitAction) ?~> "Failed to apply split action" case mergeAction: MergeAgglomerateUpdateAction => applyMergeAction(mapping, mergeAction) ?~> "Failed to apply merge action" + case revertAction: RevertToVersionUpdateAction => + revertToVersion(mapping, revertAction) ?~> "Failed to apply revert action" } private def applySplitAction(editableMappingInfo: EditableMappingInfo, update: SplitAgglomerateUpdateAction)( @@ -385,4 +387,6 @@ class EditableMappingUpdater( ) } + private def revertToVersion(mapping: EditableMappingInfo, + revertAction: RevertToVersionUpdateAction): Fox[EditableMappingInfo] = {} } From 8f1094d9377399405726660bb9f825779ed42068 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 8 Jul 2024 13:50:06 +0200 Subject: [PATCH 008/150] wip: revert editable mappings --- app/controllers/Application.scala | 15 ---- conf/webknossos.latest.routes | 1 - .../SegmentToAgglomerateProto_pb2.py | 10 +-- .../proto/SegmentToAgglomerateProto.proto | 2 +- .../annotation/DSAnnotationService.scala | 15 +--- .../EditableMappingService.scala | 7 +- .../EditableMappingStreams.scala | 71 +++++++++++++++++++ .../EditableMappingUpdater.scala | 14 ++-- 8 files changed, 91 insertions(+), 44 deletions(-) create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingStreams.scala diff --git a/app/controllers/Application.scala b/app/controllers/Application.scala index 18af7eaaef8..7d0ba191d18 100755 --- a/app/controllers/Application.scala +++ b/app/controllers/Application.scala @@ -15,11 +15,6 @@ import utils.{ApiVersioning, StoreModules, WkConf} import javax.inject.Inject import scala.concurrent.ExecutionContext -import scalapb.GeneratedMessage -import com.scalableminds.webknossos.datastore.Annotation.{ - UpdateLayerAnnotationUpdateAction, - UpdateLayerMetadataAnnotationUpdateAction -} class Application @Inject()(actorSystem: ActorSystem, userService: UserService, @@ -35,16 +30,6 @@ class Application @Inject()(actorSystem: ActorSystem, private lazy val Mailer = actorSystem.actorSelection("/user/mailActor") - def test: Action[AnyContent] = Action.async { implicit request => - Fox.successful(Ok(matchThing(UpdateLayerMetadataAnnotationUpdateAction("id", "name")))) - } - - private def matchThing(m: GeneratedMessage): String = m match { - case u: UpdateLayerAnnotationUpdateAction => "update!" - case u: UpdateLayerMetadataAnnotationUpdateAction => "other update!" - case _ => "anything else!" - } - // Note: This route is used by external applications, keep stable def buildInfo: Action[AnyContent] = sil.UserAwareAction.async { for { diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes index 2f47c9d57a1..b79feba1742 100644 --- a/conf/webknossos.latest.routes +++ b/conf/webknossos.latest.routes @@ -1,7 +1,6 @@ # Routes # This file defines all application routes (Higher priority routes first) # ~~~~ -GET /test controllers.Application.test() GET /buildinfo controllers.Application.buildInfo() GET /features controllers.Application.features() GET /health controllers.Application.health() diff --git a/tools/migrate-editable-mappings/SegmentToAgglomerateProto_pb2.py b/tools/migrate-editable-mappings/SegmentToAgglomerateProto_pb2.py index 91d2140c7b3..d7b553b51d0 100644 --- a/tools/migrate-editable-mappings/SegmentToAgglomerateProto_pb2.py +++ b/tools/migrate-editable-mappings/SegmentToAgglomerateProto_pb2.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! -# source: SegmentToAgglomerateProto.proto +# source: SegmentToAgglomerateChunkProto.proto """Generated protocol buffer code.""" from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor @@ -13,15 +13,15 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1fSegmentToAgglomerateProto.proto\x12&com.scalableminds.webknossos.datastore\"B\n\x16SegmentAgglomeratePair\x12\x11\n\tsegmentId\x18\x01 \x02(\x03\x12\x15\n\ragglomerateId\x18\x02 \x02(\x03\"y\n\x19SegmentToAgglomerateProto\x12\\\n\x14segmentToAgglomerate\x18\x01 \x03(\x0b\x32>.com.scalableminds.webknossos.datastore.SegmentAgglomeratePair') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1fSegmentToAgglomerateChunkProto.proto\x12&com.scalableminds.webknossos.datastore\"B\n\x16SegmentAgglomeratePair\x12\x11\n\tsegmentId\x18\x01 \x02(\x03\x12\x15\n\ragglomerateId\x18\x02 \x02(\x03\"y\n\x19SegmentToAgglomerateChunkProto\x12\\\n\x14segmentToAgglomerate\x18\x01 \x03(\x0b\x32>.com.scalableminds.webknossos.datastore.SegmentAgglomeratePair') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'SegmentToAgglomerateProto_pb2', globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'SegmentToAgglomerateChunkProto_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None _SEGMENTAGGLOMERATEPAIR._serialized_start=75 _SEGMENTAGGLOMERATEPAIR._serialized_end=141 - _SEGMENTTOAGGLOMERATEPROTO._serialized_start=143 - _SEGMENTTOAGGLOMERATEPROTO._serialized_end=264 + _SegmentToAgglomerateChunkProto._serialized_start=143 + _SegmentToAgglomerateChunkProto._serialized_end=264 # @@protoc_insertion_point(module_scope) diff --git a/webknossos-datastore/proto/SegmentToAgglomerateProto.proto b/webknossos-datastore/proto/SegmentToAgglomerateProto.proto index 519276323c3..6bb61fdf783 100644 --- a/webknossos-datastore/proto/SegmentToAgglomerateProto.proto +++ b/webknossos-datastore/proto/SegmentToAgglomerateProto.proto @@ -7,6 +7,6 @@ message SegmentAgglomeratePair { required int64 agglomerateId = 2; } -message SegmentToAgglomerateProto { +message SegmentToAgglomerateChunkProto { repeated SegmentAgglomeratePair segmentToAgglomerate = 1; } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala index ae60f6c65c4..f6dffa0c274 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala @@ -6,8 +6,6 @@ import com.scalableminds.webknossos.datastore.Annotation.{ AnnotationLayerProto, AnnotationProto, DeleteLayerAnnotationUpdateAction, - UpdateLayerAnnotationUpdateAction, - UpdateLayerEditableMappingAnnotationUpdateAction, UpdateLayerMetadataAnnotationUpdateAction, UpdateMetadataAnnotationUpdateAction } @@ -28,20 +26,9 @@ class DSAnnotationService @Inject()() { withAppliedChange <- updateAction match { case a: AddLayerAnnotationUpdateAction => Fox.successful( - annotation.copy( - layers = annotation.layers :+ AnnotationLayerProto(a.tracingId, - a.name, - version = 0L, - editableMappingVersion = None, - `type` = a.`type`))) + annotation.copy(layers = annotation.layers :+ AnnotationLayerProto(a.tracingId, a.name, `type` = a.`type`))) case a: DeleteLayerAnnotationUpdateAction => Fox.successful(annotation.copy(layers = annotation.layers.filter(_.tracingId != a.tracingId))) - case a: UpdateLayerAnnotationUpdateAction => - Fox.successful(annotation.copy(layers = annotation.layers.map(l => - if (l.tracingId == a.tracingId) l.copy(version = a.layerVersion) else l))) - case a: UpdateLayerEditableMappingAnnotationUpdateAction => - Fox.successful(annotation.copy(layers = annotation.layers.map(l => - if (l.tracingId == a.tracingId) l.copy(editableMappingVersion = Some(a.editableMappingVersion)) else l))) case a: UpdateLayerMetadataAnnotationUpdateAction => Fox.successful(annotation.copy(layers = annotation.layers.map(l => if (l.tracingId == a.tracingId) l.copy(name = a.name) else l))) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index b6d86171fb3..8811fcf3110 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -7,7 +7,7 @@ import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.datastore.AgglomerateGraph.AgglomerateGraph import com.scalableminds.webknossos.datastore.EditableMappingInfo.EditableMappingInfo -import com.scalableminds.webknossos.datastore.SegmentToAgglomerateProto.SegmentToAgglomerateProto +import com.scalableminds.webknossos.datastore.SegmentToAgglomerateProto.SegmentToAgglomerateChunkProto import com.scalableminds.webknossos.datastore.SkeletonTracing.{Edge, Tree, TreeTypeProto} import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing.ElementClassProto @@ -292,7 +292,6 @@ class EditableMappingService @Inject()( tracingDataStore, relyOnAgglomerateIds = pendingUpdates.length <= 1 ) - updated <- updater.applyUpdatesAndSave(closestMaterializedWithVersion.value, pendingUpdates) } yield updated } yield updatedEditableMappingInfo @@ -416,9 +415,9 @@ class EditableMappingService @Inject()( agglomerateId: Long, version: Option[Long]): Fox[Seq[(Long, Long)]] = for { - keyValuePair: VersionedKeyValuePair[SegmentToAgglomerateProto] <- tracingDataStore.editableMappingsSegmentToAgglomerate + keyValuePair: VersionedKeyValuePair[SegmentToAgglomerateChunkProto] <- tracingDataStore.editableMappingsSegmentToAgglomerate .get(segmentToAgglomerateKey(editableMappingId, agglomerateId), version, mayBeEmpty = Some(true))( - fromProtoBytes[SegmentToAgglomerateProto]) + fromProtoBytes[SegmentToAgglomerateChunkProto]) valueProto = keyValuePair.value asSequence = valueProto.segmentToAgglomerate.map(pair => pair.segmentId -> pair.agglomerateId) } yield asSequence diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingStreams.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingStreams.scala new file mode 100644 index 00000000000..1bedcb9101f --- /dev/null +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingStreams.scala @@ -0,0 +1,71 @@ +package com.scalableminds.webknossos.tracingstore.tracings.editablemapping + +import com.scalableminds.webknossos.datastore.SegmentToAgglomerateProto.SegmentToAgglomerateChunkProto +import com.scalableminds.webknossos.tracingstore.tracings.{ + FossilDBClient, + KeyValueStoreImplicits, + VersionedKeyValuePair +} +import net.liftweb.common.Full + +import scala.annotation.tailrec + +class VersionedAgglomerateToGraphIterator(prefix: String, + segmentToAgglomerateDataStore: FossilDBClient, + version: Option[Long] = None) {} + +class VersionedSegmentToAgglomerateChunkIterator(prefix: String, + segmentToAgglomerateDataStore: FossilDBClient, + version: Option[Long] = None) + extends Iterator[(String, SegmentToAgglomerateChunkProto)] + with KeyValueStoreImplicits { + private val batchSize = 64 + + private var currentStartAfterKey: Option[String] = None + private var currentBatchIterator: Iterator[VersionedKeyValuePair[Array[Byte]]] = fetchNext + private var nextBucket: Option[VersionedKeyValuePair[SegmentToAgglomerateChunkProto]] = None + + private def fetchNext: Iterator[VersionedKeyValuePair[Array[Byte]]] = + segmentToAgglomerateDataStore.getMultipleKeys(currentStartAfterKey, Some(prefix), version, Some(batchSize)).iterator + + private def fetchNextAndSave = { + currentBatchIterator = fetchNext + currentBatchIterator + } + + private def isRevertedChunk(chunkBytes: Array[Byte]): Boolean = + chunkBytes sameElements Array[Byte](0) + + @tailrec + private def getNextNonRevertedChunk: Option[VersionedKeyValuePair[SegmentToAgglomerateChunkProto]] = + if (currentBatchIterator.hasNext) { + val chunk = currentBatchIterator.next() + currentStartAfterKey = Some(chunk.key) + val chunkParsedBox = fromProtoBytes[SegmentToAgglomerateChunkProto](chunk.value) + chunkParsedBox match { + case _ if isRevertedChunk(chunk.value) => getNextNonRevertedChunk + case Full(chunkParsed) => Some(VersionedKeyValuePair(versionedKey = chunk.versionedKey, value = chunkParsed)) + case _ => getNextNonRevertedChunk + } + } else { + if (!fetchNextAndSave.hasNext) None + else getNextNonRevertedChunk + } + + override def hasNext: Boolean = + if (nextBucket.isDefined) true + else { + nextBucket = getNextNonRevertedChunk + nextBucket.isDefined + } + + override def next(): (String, SegmentToAgglomerateChunkProto) = { + val nextRes = nextBucket match { + case Some(bucket) => bucket + case None => getNextNonRevertedChunk.get + } + nextBucket = None + (nextRes.key, nextRes.value) + } + +} diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index 80a3f0151cb..8780ff4313e 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -6,7 +6,7 @@ import com.scalableminds.webknossos.datastore.AgglomerateGraph.{AgglomerateEdge, import com.scalableminds.webknossos.datastore.EditableMappingInfo.EditableMappingInfo import com.scalableminds.webknossos.datastore.SegmentToAgglomerateProto.{ SegmentAgglomeratePair, - SegmentToAgglomerateProto + SegmentToAgglomerateChunkProto } import com.scalableminds.webknossos.tracingstore.TSRemoteDatastoreClient import com.scalableminds.webknossos.tracingstore.tracings.{ @@ -64,7 +64,7 @@ class EditableMappingUpdater( private def flushSegmentToAgglomerateChunk(key: String): Fox[Unit] = { val chunk = segmentToAgglomerateBuffer(key) - val proto = SegmentToAgglomerateProto(chunk.toVector.map { segmentAgglomerateTuple => + val proto = SegmentToAgglomerateChunkProto(chunk.toVector.map { segmentAgglomerateTuple => SegmentAgglomeratePair(segmentAgglomerateTuple._1, segmentAgglomerateTuple._2) }) tracingDataStore.editableMappingsSegmentToAgglomerate.put(key, newVersion, proto.toByteArray) @@ -387,6 +387,12 @@ class EditableMappingUpdater( ) } - private def revertToVersion(mapping: EditableMappingInfo, - revertAction: RevertToVersionUpdateAction): Fox[EditableMappingInfo] = {} + private def revertToVersion(mapping: EditableMappingInfo, revertAction: RevertToVersionUpdateAction)( + implicit ec: ExecutionContext): Fox[EditableMappingInfo] = { + val segmentToAgglomerateChunkStream = new VersionedSegmentToAgglomerateChunkIterator( + editableMappingId, + tracingDataStore.editableMappingsSegmentToAgglomerate) + Fox.failure("todo") + } + } From f300febfaa04a99bd225a7fb12e865882a124551 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 9 Jul 2024 10:27:55 +0200 Subject: [PATCH 009/150] iterate on reverting editable mappings --- .../EditableMappingService.scala | 1 + .../EditableMappingStreams.scala | 84 +++++++++++++++---- .../EditableMappingUpdater.scala | 60 +++++++++---- .../tracings/volume/VolumeDataZipHelper.scala | 4 +- .../volume/VolumeTracingBucketHelper.scala | 16 ++-- .../volume/VolumeTracingDownsampling.scala | 9 +- .../volume/VolumeTracingService.scala | 2 +- .../tracings/volume/WKWBucketStreamSink.scala | 4 +- .../volume/Zarr3BucketStreamSink.scala | 4 +- 9 files changed, 134 insertions(+), 50 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index e9697e23b27..c16671a9640 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -418,6 +418,7 @@ class EditableMappingService @Inject()( keyValuePair: VersionedKeyValuePair[SegmentToAgglomerateChunkProto] <- tracingDataStore.editableMappingsSegmentToAgglomerate .get(segmentToAgglomerateKey(editableMappingId, agglomerateId), version, mayBeEmpty = Some(true))( fromProtoBytes[SegmentToAgglomerateChunkProto]) + // interpret zero-byte as Fox.empty valueProto = keyValuePair.value asSequence = valueProto.segmentToAgglomerate.map(pair => pair.segmentId -> pair.agglomerateId) } yield asSequence diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingStreams.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingStreams.scala index 1bedcb9101f..cae82ad526c 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingStreams.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingStreams.scala @@ -1,6 +1,8 @@ package com.scalableminds.webknossos.tracingstore.tracings.editablemapping +import com.scalableminds.webknossos.datastore.AgglomerateGraph.AgglomerateGraph import com.scalableminds.webknossos.datastore.SegmentToAgglomerateProto.SegmentToAgglomerateChunkProto +import com.scalableminds.webknossos.tracingstore.tracings.volume.ReversionHelper import com.scalableminds.webknossos.tracingstore.tracings.{ FossilDBClient, KeyValueStoreImplicits, @@ -12,18 +14,70 @@ import scala.annotation.tailrec class VersionedAgglomerateToGraphIterator(prefix: String, segmentToAgglomerateDataStore: FossilDBClient, - version: Option[Long] = None) {} + version: Option[Long] = None) + extends Iterator[(String, AgglomerateGraph, Long)] + with ReversionHelper + with KeyValueStoreImplicits { + private val batchSize = 64 + + private var currentStartAfterKey: Option[String] = None + private var currentBatchIterator: Iterator[VersionedKeyValuePair[Array[Byte]]] = fetchNext + private var nextGraph: Option[VersionedKeyValuePair[AgglomerateGraph]] = None + + private def fetchNext: Iterator[VersionedKeyValuePair[Array[Byte]]] = + segmentToAgglomerateDataStore.getMultipleKeys(currentStartAfterKey, Some(prefix), version, Some(batchSize)).iterator + + private def fetchNextAndSave = { + currentBatchIterator = fetchNext + currentBatchIterator + } + + @tailrec + private def getNextNonRevertedGraph: Option[VersionedKeyValuePair[AgglomerateGraph]] = + if (currentBatchIterator.hasNext) { + val chunk = currentBatchIterator.next() + currentStartAfterKey = Some(chunk.key) + val graphParsedBox = fromProtoBytes[AgglomerateGraph](chunk.value) + graphParsedBox match { + case _ if isRevertedElement(chunk.value) => getNextNonRevertedGraph + case Full(graphParsed) => Some(VersionedKeyValuePair(versionedKey = chunk.versionedKey, value = graphParsed)) + case _ => getNextNonRevertedGraph + } + } else { + if (!fetchNextAndSave.hasNext) None + else getNextNonRevertedGraph + } + + override def hasNext: Boolean = + if (nextGraph.isDefined) true + else { + nextGraph = getNextNonRevertedGraph + nextGraph.isDefined + } + + override def next(): (String, AgglomerateGraph, Long) = { + val nextRes = nextGraph match { + case Some(bucket) => bucket + case None => getNextNonRevertedGraph.get + } + nextGraph = None + // TODO: parse graph key? (=agglomerate id) + (nextRes.key, nextRes.value, nextRes.version) + } + +} class VersionedSegmentToAgglomerateChunkIterator(prefix: String, segmentToAgglomerateDataStore: FossilDBClient, version: Option[Long] = None) - extends Iterator[(String, SegmentToAgglomerateChunkProto)] + extends Iterator[(String, SegmentToAgglomerateChunkProto, Long)] + with ReversionHelper with KeyValueStoreImplicits { private val batchSize = 64 private var currentStartAfterKey: Option[String] = None private var currentBatchIterator: Iterator[VersionedKeyValuePair[Array[Byte]]] = fetchNext - private var nextBucket: Option[VersionedKeyValuePair[SegmentToAgglomerateChunkProto]] = None + private var nextChunk: Option[VersionedKeyValuePair[SegmentToAgglomerateChunkProto]] = None private def fetchNext: Iterator[VersionedKeyValuePair[Array[Byte]]] = segmentToAgglomerateDataStore.getMultipleKeys(currentStartAfterKey, Some(prefix), version, Some(batchSize)).iterator @@ -33,9 +87,6 @@ class VersionedSegmentToAgglomerateChunkIterator(prefix: String, currentBatchIterator } - private def isRevertedChunk(chunkBytes: Array[Byte]): Boolean = - chunkBytes sameElements Array[Byte](0) - @tailrec private def getNextNonRevertedChunk: Option[VersionedKeyValuePair[SegmentToAgglomerateChunkProto]] = if (currentBatchIterator.hasNext) { @@ -43,9 +94,9 @@ class VersionedSegmentToAgglomerateChunkIterator(prefix: String, currentStartAfterKey = Some(chunk.key) val chunkParsedBox = fromProtoBytes[SegmentToAgglomerateChunkProto](chunk.value) chunkParsedBox match { - case _ if isRevertedChunk(chunk.value) => getNextNonRevertedChunk - case Full(chunkParsed) => Some(VersionedKeyValuePair(versionedKey = chunk.versionedKey, value = chunkParsed)) - case _ => getNextNonRevertedChunk + case _ if isRevertedElement(chunk.value) => getNextNonRevertedChunk + case Full(chunkParsed) => Some(VersionedKeyValuePair(versionedKey = chunk.versionedKey, value = chunkParsed)) + case _ => getNextNonRevertedChunk } } else { if (!fetchNextAndSave.hasNext) None @@ -53,19 +104,20 @@ class VersionedSegmentToAgglomerateChunkIterator(prefix: String, } override def hasNext: Boolean = - if (nextBucket.isDefined) true + if (nextChunk.isDefined) true else { - nextBucket = getNextNonRevertedChunk - nextBucket.isDefined + nextChunk = getNextNonRevertedChunk + nextChunk.isDefined } - override def next(): (String, SegmentToAgglomerateChunkProto) = { - val nextRes = nextBucket match { + override def next(): (String, SegmentToAgglomerateChunkProto, Long) = { + val nextRes = nextChunk match { case Some(bucket) => bucket case None => getNextNonRevertedChunk.get } - nextBucket = None - (nextRes.key, nextRes.value) + nextChunk = None + // TODO: parse chunk key? + (nextRes.key, nextRes.value, nextRes.version) } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index 8780ff4313e..22ad3a7bdf1 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -43,10 +43,12 @@ class EditableMappingUpdater( with FoxImplicits with LazyLogging { - private val segmentToAgglomerateBuffer: mutable.Map[String, Map[Long, Long]] = - new mutable.HashMap[String, Map[Long, Long]]() - private val agglomerateToGraphBuffer: mutable.Map[String, AgglomerateGraph] = - new mutable.HashMap[String, AgglomerateGraph]() + // chunkKey → (Map[segmentId → agglomerateId], isToBeReverted) + private val segmentToAgglomerateBuffer: mutable.Map[String, (Map[Long, Long], Boolean)] = + new mutable.HashMap[String, (Map[Long, Long], Boolean)]() + // agglomerateKey → (agglomerateGraph, isToBeReverted) + private val agglomerateToGraphBuffer: mutable.Map[String, (AgglomerateGraph, Boolean)] = + new mutable.HashMap[String, (AgglomerateGraph, Boolean)]() def applyUpdatesAndSave(existingEditabeMappingInfo: EditableMappingInfo, updates: List[EditableMappingUpdateAction])( implicit ec: ExecutionContext): Fox[EditableMappingInfo] = @@ -63,7 +65,8 @@ class EditableMappingUpdater( } yield () private def flushSegmentToAgglomerateChunk(key: String): Fox[Unit] = { - val chunk = segmentToAgglomerateBuffer(key) + val (chunk, isToBeReverted) = segmentToAgglomerateBuffer(key) + // TODO respect isToBeReverted val proto = SegmentToAgglomerateChunkProto(chunk.toVector.map { segmentAgglomerateTuple => SegmentAgglomeratePair(segmentAgglomerateTuple._1, segmentAgglomerateTuple._2) }) @@ -71,7 +74,8 @@ class EditableMappingUpdater( } private def flushAgglomerateGraph(key: String): Fox[Unit] = { - val graph = agglomerateToGraphBuffer(key) + val (graph, isToBeReverted) = agglomerateToGraphBuffer(key) + // TODO respect isToBeReverted tracingDataStore.editableMappingsAgglomerateToGraph.put(key, newVersion, graph) } @@ -104,7 +108,7 @@ class EditableMappingUpdater( case mergeAction: MergeAgglomerateUpdateAction => applyMergeAction(mapping, mergeAction) ?~> "Failed to apply merge action" case revertAction: RevertToVersionUpdateAction => - revertToVersion(mapping, revertAction) ?~> "Failed to apply revert action" + revertToVersion(revertAction) ?~> "Failed to apply revert action" } private def applySplitAction(editableMappingInfo: EditableMappingInfo, update: SplitAgglomerateUpdateAction)( @@ -160,6 +164,7 @@ class EditableMappingUpdater( val chunkId = segmentId / editableMappingService.defaultSegmentToAgglomerateChunkSize val chunkKey = editableMappingService.segmentToAgglomerateKey(editableMappingId, chunkId) val chunkFromBufferOpt = segmentToAgglomerateBuffer.get(chunkKey) + // TODO isToBeReverted → None for { chunk <- Fox.fillOption(chunkFromBufferOpt) { editableMappingService @@ -192,13 +197,14 @@ class EditableMappingUpdater( existingChunk: Map[Long, Long] <- getSegmentToAgglomerateChunkWithEmptyFallback(editableMappingId, chunkId) ?~> "failed to get old segment to agglomerate chunk for updating it" mergedMap = existingChunk ++ segmentIdsToUpdate.map(_ -> agglomerateId).toMap _ = segmentToAgglomerateBuffer.put(editableMappingService.segmentToAgglomerateKey(editableMappingId, chunkId), - mergedMap) + (mergedMap, false)) } yield () private def getSegmentToAgglomerateChunkWithEmptyFallback(editableMappingId: String, chunkId: Long)( implicit ec: ExecutionContext): Fox[Map[Long, Long]] = { val key = editableMappingService.segmentToAgglomerateKey(editableMappingId, chunkId) val fromBufferOpt = segmentToAgglomerateBuffer.get(key) + // TODO isToBeReverted → None Fox.fillOption(fromBufferOpt) { editableMappingService .getSegmentToAgglomerateChunkWithEmptyFallback(editableMappingId, chunkId, version = oldVersion) @@ -210,6 +216,7 @@ class EditableMappingUpdater( implicit ec: ExecutionContext): Fox[AgglomerateGraph] = { val key = editableMappingService.agglomerateGraphKey(editableMappingId, agglomerateId) val fromBufferOpt = agglomerateToGraphBuffer.get(key) + // TODO isToBeReverted → None fromBufferOpt.map(Fox.successful(_)).getOrElse { editableMappingService.getAgglomerateGraphForIdWithFallback(mapping, editableMappingId, @@ -222,7 +229,7 @@ class EditableMappingUpdater( private def updateAgglomerateGraph(agglomerateId: Long, graph: AgglomerateGraph): Unit = { val key = editableMappingService.agglomerateGraphKey(editableMappingId, agglomerateId) - agglomerateToGraphBuffer.put(key, graph) + agglomerateToGraphBuffer.put(key, (graph, false)) } private def splitGraph(agglomerateId: Long, @@ -387,12 +394,33 @@ class EditableMappingUpdater( ) } - private def revertToVersion(mapping: EditableMappingInfo, revertAction: RevertToVersionUpdateAction)( - implicit ec: ExecutionContext): Fox[EditableMappingInfo] = { - val segmentToAgglomerateChunkStream = new VersionedSegmentToAgglomerateChunkIterator( - editableMappingId, - tracingDataStore.editableMappingsSegmentToAgglomerate) - Fox.failure("todo") - } + private def revertToVersion(revertAction: RevertToVersionUpdateAction)( + implicit ec: ExecutionContext): Fox[EditableMappingInfo] = + for { + _ <- bool2Fox(revertAction.sourceVersion >= oldVersion) ?~> "trying to revert editable mapping to a version not yet present in the database" + oldInfo <- editableMappingService.getInfo(editableMappingId, + Some(revertAction.sourceVersion), + remoteFallbackLayer, + userToken) + _ = segmentToAgglomerateBuffer.clear() + _ = agglomerateToGraphBuffer.clear() + segmentToAgglomerateChunkNewestStream = new VersionedSegmentToAgglomerateChunkIterator( + editableMappingId, + tracingDataStore.editableMappingsSegmentToAgglomerate) + _ <- Fox.serialCombined(segmentToAgglomerateChunkNewestStream) { + case (chunkKey, chunkDataBeforeRevert, version) => + if (version > revertAction.sourceVersion) { + // TODO fetch old chunk, save to buffer. if empty, write zero-byte + case Empty => Fox.successful(()) // TODO save zero-byte + case Failure(msg, _, chain) => + Fox.failure(msg, Empty, chain) + Fox.successful(()) + } else Fox.successful(()) + } + agglomerateToGraphNewestStream = new VersionedAgglomerateToGraphIterator( + editableMappingId, + tracingDataStore.editableMappingsAgglomerateToGraph) + // TODO do we need to iterate over old *and* new to get all additions + removals? + } yield oldInfo } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeDataZipHelper.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeDataZipHelper.scala index ce74001a669..35aacb77e98 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeDataZipHelper.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeDataZipHelper.scala @@ -27,7 +27,7 @@ import scala.concurrent.ExecutionContext trait VolumeDataZipHelper extends WKWDataFormatHelper - with VolumeBucketReversionHelper + with ReversionHelper with BoxImplicits with LazyLogging { @@ -61,7 +61,7 @@ trait VolumeDataZipHelper parseWKWFilePath(fileName.toString).map { bucketPosition: BucketPosition => if (buckets.hasNext) { val data = buckets.next() - if (!isRevertedBucket(data)) { + if (!isRevertedElement(data)) { block(bucketPosition, data) } else Fox.successful(()) } else Fox.successful(()) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala index 97d64b826db..3aa03f8be6d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala @@ -15,10 +15,12 @@ import scala.annotation.tailrec import scala.concurrent.ExecutionContext import scala.concurrent.duration._ -trait VolumeBucketReversionHelper { - protected def isRevertedBucket(data: Array[Byte]): Boolean = data sameElements Array[Byte](0) +trait ReversionHelper { + val revertedValue: Array[Byte] = Array[Byte](0) - protected def isRevertedBucket(bucket: VersionedKeyValuePair[Array[Byte]]): Boolean = isRevertedBucket(bucket.value) + protected def isRevertedElement(data: Array[Byte]): Boolean = data.sameElements(revertedValue) + + protected def isRevertedElement(bucket: VersionedKeyValuePair[Array[Byte]]): Boolean = isRevertedElement(bucket.value) } trait VolumeBucketCompression extends LazyLogging { @@ -173,7 +175,7 @@ trait VolumeTracingBucketHelper with VolumeBucketCompression with DataConverter with BucketKeys - with VolumeBucketReversionHelper { + with ReversionHelper { implicit def ec: ExecutionContext @@ -196,7 +198,7 @@ trait VolumeTracingBucketHelper case None => volumeDataStore.get(key, version, mayBeEmpty = Some(true)) } val unpackedDataFox = dataFox.flatMap { versionedVolumeBucket => - if (isRevertedBucket(versionedVolumeBucket)) Fox.empty + if (isRevertedElement(versionedVolumeBucket)) Fox.empty else { val debugInfo = s"key: $key, ${versionedVolumeBucket.value.length} bytes, version ${versionedVolumeBucket.version}" @@ -304,7 +306,7 @@ class VersionedBucketIterator(prefix: String, with VolumeBucketCompression with BucketKeys with FoxImplicits - with VolumeBucketReversionHelper { + with ReversionHelper { private val batchSize = 64 private var currentStartAfterKey: Option[String] = None @@ -324,7 +326,7 @@ class VersionedBucketIterator(prefix: String, if (currentBatchIterator.hasNext) { val bucket = currentBatchIterator.next() currentStartAfterKey = Some(bucket.key) - if (isRevertedBucket(bucket) || parseBucketKey(bucket.key, additionalAxes).isEmpty) { + if (isRevertedElement(bucket) || parseBucketKey(bucket.key, additionalAxes).isEmpty) { getNextNonRevertedBucket } else { Some(bucket) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala index b5c9eec46c2..bb2dd9f18a1 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala @@ -52,6 +52,7 @@ trait VolumeTracingDownsampling with ProtoGeometryImplicits with VolumeBucketCompression with KeyValueStoreImplicits + with ReversionHelper with FoxImplicits { val tracingDataStore: TracingDataStore @@ -89,7 +90,7 @@ trait VolumeTracingDownsampling sourceMag = getSourceMag(tracing) magsToCreate <- getMagsToCreate(tracing, oldTracingId) elementClass = elementClassFromProto(tracing.elementClass) - bucketDataMapMutable = new mutable.HashMap[BucketPosition, Array[Byte]]().withDefault(_ => Array[Byte](0)) + bucketDataMapMutable = new mutable.HashMap[BucketPosition, Array[Byte]]().withDefault(_ => revertedValue) _ = fillMapWithSourceBucketsInplace(bucketDataMapMutable, tracingId, dataLayer, sourceMag) originalBucketPositions = bucketDataMapMutable.keys.toList updatedBucketsMutable = new mutable.ListBuffer[BucketPosition]() @@ -167,8 +168,8 @@ trait VolumeTracingDownsampling sourceBucketPositionsFor(downsampledBucketPosition, downScaleFactor, previousMag) val sourceData: Seq[Array[Byte]] = sourceBuckets.map(bucketDataMapMutable(_)) val downsampledData: Array[Byte] = - if (sourceData.forall(_.sameElements(Array[Byte](0)))) - Array[Byte](0) + if (sourceData.forall(_.sameElements(revertedValue))) + revertedValue else { val sourceDataFilled = fillZeroedIfNeeded(sourceData, bucketVolume, dataLayer.bytesPerElement) val sourceDataTyped = UnsignedIntegerArray.fromByteArray(sourceDataFilled.toArray.flatten, elementClass) @@ -216,7 +217,7 @@ trait VolumeTracingDownsampling // Reverted buckets and missing buckets are represented by a single zero-byte. // For downsampling, those need to be replaced with the full bucket volume of zero-bytes. sourceData.map { sourceBucketData => - if (sourceBucketData.sameElements(Array[Byte](0))) { + if (isRevertedElement(sourceBucketData)) { Array.fill[Byte](bucketVolume * bytesPerElement)(0) } else sourceBucketData } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 5fa306a9d2c..74c46726eae 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -330,7 +330,7 @@ class VolumeTracingService @Inject()( } yield () case Empty => for { - dataAfterRevert <- Fox.successful(Array[Byte](0)) + dataAfterRevert <- Fox.successful(revertedValue) _ <- saveBucket(dataLayer, bucketPosition, dataAfterRevert, newVersion) _ <- Fox.runIfOptionTrue(tracing.hasSegmentIndex)( updateSegmentIndex( diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/WKWBucketStreamSink.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/WKWBucketStreamSink.scala index ec8df4f8f59..b1783fe8d9d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/WKWBucketStreamSink.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/WKWBucketStreamSink.scala @@ -18,7 +18,7 @@ import scala.concurrent.{ExecutionContext, Future} class WKWBucketStreamSink(val layer: DataLayer, tracingHasFallbackLayer: Boolean) extends WKWDataFormatHelper - with VolumeBucketReversionHelper + with ReversionHelper with ByteUtils { def apply(bucketStream: Iterator[(BucketPosition, Array[Byte])], mags: Seq[Vec3Int])( @@ -27,7 +27,7 @@ class WKWBucketStreamSink(val layer: DataLayer, tracingHasFallbackLayer: Boolean val header = WKWHeader(1, DataLayer.bucketLength, ChunkType.LZ4, voxelType, numChannels) bucketStream.flatMap { case (bucket, data) => - val skipBucket = if (tracingHasFallbackLayer) isRevertedBucket(data) else isAllZero(data) + val skipBucket = if (tracingHasFallbackLayer) isRevertedElement(data) else isAllZero(data) if (skipBucket) { // If the tracing has no fallback segmentation, all-zero buckets can be omitted entirely None diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala index 2d1d024ace4..ac85fd86b5c 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/Zarr3BucketStreamSink.scala @@ -27,7 +27,7 @@ import scala.concurrent.{ExecutionContext, Future} // Creates data zip from volume tracings class Zarr3BucketStreamSink(val layer: VolumeTracingLayer, tracingHasFallbackLayer: Boolean) extends ProtoGeometryImplicits - with VolumeBucketReversionHelper + with ReversionHelper with ByteUtils { private lazy val defaultLayerName = "volumeAnnotationData" @@ -74,7 +74,7 @@ class Zarr3BucketStreamSink(val layer: VolumeTracingLayer, tracingHasFallbackLay ) bucketStream.flatMap { case (bucket, data) => - val skipBucket = if (tracingHasFallbackLayer) isAllZero(data) else isRevertedBucket(data) + val skipBucket = if (tracingHasFallbackLayer) isAllZero(data) else isRevertedElement(data) if (skipBucket) { // If the tracing has no fallback segmentation, all-zero buckets can be omitted entirely None From 9f99feb2edb29f8bc801f2b0eab38c46ee31e70c Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 9 Jul 2024 11:43:57 +0200 Subject: [PATCH 010/150] revert agglomerateToGraph + segmentToAgglomerate in buffers --- .../EditableMappingService.scala | 30 ++++--- .../EditableMappingUpdater.scala | 81 ++++++++++++++----- 2 files changed, 80 insertions(+), 31 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index c16671a9640..3b3ca98d6bf 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -21,6 +21,7 @@ import com.scalableminds.webknossos.datastore.services.{ AdHocMeshServiceHolder, BinaryDataService } +import com.scalableminds.webknossos.tracingstore.tracings.volume.ReversionHelper import com.scalableminds.webknossos.tracingstore.tracings.{ FallbackDataHelper, KeyValueStoreImplicits, @@ -96,6 +97,7 @@ class EditableMappingService @Inject()( extends KeyValueStoreImplicits with FallbackDataHelper with FoxImplicits + with ReversionHelper with LazyLogging with ProtoGeometryImplicits { @@ -413,13 +415,19 @@ class EditableMappingService @Inject()( private def getSegmentToAgglomerateChunk(editableMappingId: String, agglomerateId: Long, - version: Option[Long]): Fox[Seq[(Long, Long)]] = + version: Option[Long]): Fox[Seq[(Long, Long)]] = { + val chunkKey = segmentToAgglomerateKey(editableMappingId, agglomerateId) + getSegmentToAgglomerateChunk(editableMappingId, chunkKey, version) + } + + def getSegmentToAgglomerateChunk(editableMappingId: String, + chunkKey: String, + version: Option[Long]): Fox[Seq[(Long, Long)]] = for { - keyValuePair: VersionedKeyValuePair[SegmentToAgglomerateChunkProto] <- tracingDataStore.editableMappingsSegmentToAgglomerate - .get(segmentToAgglomerateKey(editableMappingId, agglomerateId), version, mayBeEmpty = Some(true))( - fromProtoBytes[SegmentToAgglomerateChunkProto]) - // interpret zero-byte as Fox.empty - valueProto = keyValuePair.value + keyValuePairBytes: VersionedKeyValuePair[Array[Byte]] <- tracingDataStore.editableMappingsSegmentToAgglomerate + .get(chunkKey, version, mayBeEmpty = Some(true)) + valueProto <- if (isRevertedElement(keyValuePairBytes.value)) Fox.empty + else fromProtoBytes[SegmentToAgglomerateChunkProto](keyValuePairBytes.value).toFox asSequence = valueProto.segmentToAgglomerate.map(pair => pair.segmentId -> pair.agglomerateId) } yield asSequence @@ -600,10 +608,12 @@ class EditableMappingService @Inject()( agglomerateGraph <- agglomerateToGraphCache.getOrLoad( (mappingId, agglomerateId, version), _ => - tracingDataStore.editableMappingsAgglomerateToGraph - .get(agglomerateGraphKey(mappingId, agglomerateId), Some(version), mayBeEmpty = Some(true))( - fromProtoBytes[AgglomerateGraph]) - .map(_.value) + for { + graphBytes: VersionedKeyValuePair[Array[Byte]] <- tracingDataStore.editableMappingsAgglomerateToGraph + .get(agglomerateGraphKey(mappingId, agglomerateId), Some(version), mayBeEmpty = Some(true)) + graphParsed <- if (isRevertedElement(graphBytes.value)) Fox.empty + else fromProtoBytes[AgglomerateGraph](graphBytes.value).toFox + } yield graphParsed ) } yield agglomerateGraph diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index 22ad3a7bdf1..564b2c41df8 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -9,6 +9,7 @@ import com.scalableminds.webknossos.datastore.SegmentToAgglomerateProto.{ SegmentToAgglomerateChunkProto } import com.scalableminds.webknossos.tracingstore.TSRemoteDatastoreClient +import com.scalableminds.webknossos.tracingstore.tracings.volume.ReversionHelper import com.scalableminds.webknossos.tracingstore.tracings.{ KeyValueStoreImplicits, RemoteFallbackLayer, @@ -40,6 +41,7 @@ class EditableMappingUpdater( tracingDataStore: TracingDataStore, relyOnAgglomerateIds: Boolean // False during merge and in case of multiple actions. Then, look up all agglomerate ids at positions ) extends KeyValueStoreImplicits + with ReversionHelper with FoxImplicits with LazyLogging { @@ -66,17 +68,21 @@ class EditableMappingUpdater( private def flushSegmentToAgglomerateChunk(key: String): Fox[Unit] = { val (chunk, isToBeReverted) = segmentToAgglomerateBuffer(key) - // TODO respect isToBeReverted - val proto = SegmentToAgglomerateChunkProto(chunk.toVector.map { segmentAgglomerateTuple => - SegmentAgglomeratePair(segmentAgglomerateTuple._1, segmentAgglomerateTuple._2) - }) - tracingDataStore.editableMappingsSegmentToAgglomerate.put(key, newVersion, proto.toByteArray) + val valueToFlush: Array[Byte] = + if (isToBeReverted) revertedValue + else { + val proto = SegmentToAgglomerateChunkProto(chunk.toVector.map { segmentAgglomerateTuple => + SegmentAgglomeratePair(segmentAgglomerateTuple._1, segmentAgglomerateTuple._2) + }) + proto.toByteArray + } + tracingDataStore.editableMappingsSegmentToAgglomerate.put(key, newVersion, valueToFlush) } private def flushAgglomerateGraph(key: String): Fox[Unit] = { val (graph, isToBeReverted) = agglomerateToGraphBuffer(key) - // TODO respect isToBeReverted - tracingDataStore.editableMappingsAgglomerateToGraph.put(key, newVersion, graph) + val valueToFlush: Array[Byte] = if (isToBeReverted) revertedValue else graph + tracingDataStore.editableMappingsAgglomerateToGraph.put(key, newVersion, valueToFlush) } private def updateIter(mappingFox: Fox[EditableMappingInfo], remainingUpdates: List[EditableMappingUpdateAction])( @@ -160,11 +166,22 @@ class EditableMappingUpdater( } yield (agglomerateId1, agglomerateId2) } + private def getFromSegmentToAgglomerateBuffer(chunkKey: String): Option[Map[Long, Long]] = + segmentToAgglomerateBuffer.get(chunkKey).flatMap { + case (chunkFromBuffer, isToBeReverted) => + if (isToBeReverted) None else Some(chunkFromBuffer) + } + + private def getFromAgglomerateToGraphBuffer(chunkKey: String): Option[AgglomerateGraph] = + agglomerateToGraphBuffer.get(chunkKey).flatMap { + case (graphFromBuffer, isToBeReverted) => + if (isToBeReverted) None else Some(graphFromBuffer) + } + private def agglomerateIdForSegmentId(segmentId: Long)(implicit ec: ExecutionContext): Fox[Long] = { val chunkId = segmentId / editableMappingService.defaultSegmentToAgglomerateChunkSize val chunkKey = editableMappingService.segmentToAgglomerateKey(editableMappingId, chunkId) - val chunkFromBufferOpt = segmentToAgglomerateBuffer.get(chunkKey) - // TODO isToBeReverted → None + val chunkFromBufferOpt = getFromSegmentToAgglomerateBuffer(chunkKey) for { chunk <- Fox.fillOption(chunkFromBufferOpt) { editableMappingService @@ -203,8 +220,7 @@ class EditableMappingUpdater( private def getSegmentToAgglomerateChunkWithEmptyFallback(editableMappingId: String, chunkId: Long)( implicit ec: ExecutionContext): Fox[Map[Long, Long]] = { val key = editableMappingService.segmentToAgglomerateKey(editableMappingId, chunkId) - val fromBufferOpt = segmentToAgglomerateBuffer.get(key) - // TODO isToBeReverted → None + val fromBufferOpt = getFromSegmentToAgglomerateBuffer(key) Fox.fillOption(fromBufferOpt) { editableMappingService .getSegmentToAgglomerateChunkWithEmptyFallback(editableMappingId, chunkId, version = oldVersion) @@ -215,8 +231,7 @@ class EditableMappingUpdater( private def agglomerateGraphForIdWithFallback(mapping: EditableMappingInfo, agglomerateId: Long)( implicit ec: ExecutionContext): Fox[AgglomerateGraph] = { val key = editableMappingService.agglomerateGraphKey(editableMappingId, agglomerateId) - val fromBufferOpt = agglomerateToGraphBuffer.get(key) - // TODO isToBeReverted → None + val fromBufferOpt = getFromAgglomerateToGraphBuffer(key) fromBufferOpt.map(Fox.successful(_)).getOrElse { editableMappingService.getAgglomerateGraphForIdWithFallback(mapping, editableMappingId, @@ -232,6 +247,8 @@ class EditableMappingUpdater( agglomerateToGraphBuffer.put(key, (graph, false)) } + private def emptyAgglomerateGraph = AgglomerateGraph(Seq(), Seq(), Seq(), Seq()) + private def splitGraph(agglomerateId: Long, agglomerateGraph: AgglomerateGraph, update: SplitAgglomerateUpdateAction, @@ -247,7 +264,7 @@ class EditableMappingUpdater( logger.warn( s"Split action for editable mapping $editableMappingId: Edge to remove ($segmentId1 at ${update.segmentPosition1} in mag ${update.mag} to $segmentId2 at ${update.segmentPosition2} in mag ${update.mag} in agglomerate $agglomerateId) already absent. This split becomes a no-op.") } - (agglomerateGraph, AgglomerateGraph(Seq(), Seq(), Seq(), Seq())) + (agglomerateGraph, emptyAgglomerateGraph) } else { val graph1Nodes: Set[Long] = computeConnectedComponent(startNode = segmentId1, @@ -408,19 +425,41 @@ class EditableMappingUpdater( editableMappingId, tracingDataStore.editableMappingsSegmentToAgglomerate) _ <- Fox.serialCombined(segmentToAgglomerateChunkNewestStream) { - case (chunkKey, chunkDataBeforeRevert, version) => + case (chunkKey, _, version) => if (version > revertAction.sourceVersion) { - // TODO fetch old chunk, save to buffer. if empty, write zero-byte - case Empty => Fox.successful(()) // TODO save zero-byte - case Failure(msg, _, chain) => - Fox.failure(msg, Empty, chain) - Fox.successful(()) + editableMappingService + .getSegmentToAgglomerateChunk(editableMappingId, chunkKey, Some(revertAction.sourceVersion)) + .futureBox + .map { + case Full(chunkData) => segmentToAgglomerateBuffer.put(chunkKey, (chunkData.toMap, false)) + case Empty => segmentToAgglomerateBuffer.put(chunkKey, (Map[Long, Long](), true)) + case Failure(msg, _, chain) => + Fox.failure(msg, Empty, chain) + } } else Fox.successful(()) } agglomerateToGraphNewestStream = new VersionedAgglomerateToGraphIterator( editableMappingId, tracingDataStore.editableMappingsAgglomerateToGraph) - // TODO do we need to iterate over old *and* new to get all additions + removals? + _ <- Fox.serialCombined(agglomerateToGraphNewestStream) { + case (graphKey, _, version) => + if (version > revertAction.sourceVersion) { + // TODO agglomerate id from graph key + editableMappingService + .getAgglomerateGraphForId(editableMappingId, + 0L, + remoteFallbackLayer, + userToken, + Some(revertAction.sourceVersion)) + .futureBox + .map { + case Full(graphData) => agglomerateToGraphBuffer.put(graphKey, (graphData, false)) + case Empty => agglomerateToGraphBuffer.put(graphKey, (emptyAgglomerateGraph, true)) + case Failure(msg, _, chain) => + Fox.failure(msg, Empty, chain) + } + } else Fox.successful(()) + } } yield oldInfo } From 130e1f75a2bfca399d3154fb405ba43b6148e7d3 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 9 Jul 2024 11:51:22 +0200 Subject: [PATCH 011/150] use right key --- .../EditableMappingElementKeys.scala | 18 ++++++++ .../EditableMappingService.scala | 17 ++----- .../EditableMappingUpdater.scala | 44 ++++++++++--------- 3 files changed, 44 insertions(+), 35 deletions(-) create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingElementKeys.scala diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingElementKeys.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingElementKeys.scala new file mode 100644 index 00000000000..af9f7a2a287 --- /dev/null +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingElementKeys.scala @@ -0,0 +1,18 @@ +package com.scalableminds.webknossos.tracingstore.tracings.editablemapping + +import net.liftweb.common.Box +import net.liftweb.common.Box.tryo + +trait EditableMappingElementKeys { + + protected def agglomerateGraphKey(mappingId: String, agglomerateId: Long): String = + s"$mappingId/$agglomerateId" + + protected def segmentToAgglomerateKey(mappingId: String, chunkId: Long): String = + s"$mappingId/$chunkId" + + protected def chunkIdFromSegmentToAgglomerateKey(key: String): Box[Long] = tryo(key.split("/")(1).toLong) + + protected def agglomerateIdFromAgglomerateGraphKey(key: String): Box[Long] = tryo(key.split("/")(1).toLong) + +} diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index 3b3ca98d6bf..39fe85eb066 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -98,6 +98,7 @@ class EditableMappingService @Inject()( with FallbackDataHelper with FoxImplicits with ReversionHelper + with EditableMappingElementKeys with LazyLogging with ProtoGeometryImplicits { @@ -417,12 +418,10 @@ class EditableMappingService @Inject()( agglomerateId: Long, version: Option[Long]): Fox[Seq[(Long, Long)]] = { val chunkKey = segmentToAgglomerateKey(editableMappingId, agglomerateId) - getSegmentToAgglomerateChunk(editableMappingId, chunkKey, version) + getSegmentToAgglomerateChunk(chunkKey, version) } - def getSegmentToAgglomerateChunk(editableMappingId: String, - chunkKey: String, - version: Option[Long]): Fox[Seq[(Long, Long)]] = + def getSegmentToAgglomerateChunk(chunkKey: String, version: Option[Long]): Fox[Seq[(Long, Long)]] = for { keyValuePairBytes: VersionedKeyValuePair[Array[Byte]] <- tracingDataStore.editableMappingsSegmentToAgglomerate .get(chunkKey, version, mayBeEmpty = Some(true)) @@ -587,16 +586,6 @@ class EditableMappingService @Inject()( result <- adHocMeshService.requestAdHocMeshViaActor(adHocMeshRequest) } yield result - def agglomerateGraphKey(mappingId: String, agglomerateId: Long): String = - s"$mappingId/$agglomerateId" - - def segmentToAgglomerateKey(mappingId: String, chunkId: Long): String = - s"$mappingId/$chunkId" - - private def chunkIdFromSegmentToAgglomerateKey(key: String): Box[Long] = tryo(key.split("/")(1).toLong) - - private def agglomerateIdFromAgglomerateGraphKey(key: String): Box[Long] = tryo(key.split("/")(1).toLong) - def getAgglomerateGraphForId(mappingId: String, agglomerateId: Long, remoteFallbackLayer: RemoteFallbackLayer, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index 564b2c41df8..55b97b96e8d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -43,6 +43,7 @@ class EditableMappingUpdater( ) extends KeyValueStoreImplicits with ReversionHelper with FoxImplicits + with EditableMappingElementKeys with LazyLogging { // chunkKey → (Map[segmentId → agglomerateId], isToBeReverted) @@ -180,7 +181,7 @@ class EditableMappingUpdater( private def agglomerateIdForSegmentId(segmentId: Long)(implicit ec: ExecutionContext): Fox[Long] = { val chunkId = segmentId / editableMappingService.defaultSegmentToAgglomerateChunkSize - val chunkKey = editableMappingService.segmentToAgglomerateKey(editableMappingId, chunkId) + val chunkKey = segmentToAgglomerateKey(editableMappingId, chunkId) val chunkFromBufferOpt = getFromSegmentToAgglomerateBuffer(chunkKey) for { chunk <- Fox.fillOption(chunkFromBufferOpt) { @@ -213,13 +214,12 @@ class EditableMappingUpdater( for { existingChunk: Map[Long, Long] <- getSegmentToAgglomerateChunkWithEmptyFallback(editableMappingId, chunkId) ?~> "failed to get old segment to agglomerate chunk for updating it" mergedMap = existingChunk ++ segmentIdsToUpdate.map(_ -> agglomerateId).toMap - _ = segmentToAgglomerateBuffer.put(editableMappingService.segmentToAgglomerateKey(editableMappingId, chunkId), - (mergedMap, false)) + _ = segmentToAgglomerateBuffer.put(segmentToAgglomerateKey(editableMappingId, chunkId), (mergedMap, false)) } yield () private def getSegmentToAgglomerateChunkWithEmptyFallback(editableMappingId: String, chunkId: Long)( implicit ec: ExecutionContext): Fox[Map[Long, Long]] = { - val key = editableMappingService.segmentToAgglomerateKey(editableMappingId, chunkId) + val key = segmentToAgglomerateKey(editableMappingId, chunkId) val fromBufferOpt = getFromSegmentToAgglomerateBuffer(key) Fox.fillOption(fromBufferOpt) { editableMappingService @@ -230,7 +230,7 @@ class EditableMappingUpdater( private def agglomerateGraphForIdWithFallback(mapping: EditableMappingInfo, agglomerateId: Long)( implicit ec: ExecutionContext): Fox[AgglomerateGraph] = { - val key = editableMappingService.agglomerateGraphKey(editableMappingId, agglomerateId) + val key = agglomerateGraphKey(editableMappingId, agglomerateId) val fromBufferOpt = getFromAgglomerateToGraphBuffer(key) fromBufferOpt.map(Fox.successful(_)).getOrElse { editableMappingService.getAgglomerateGraphForIdWithFallback(mapping, @@ -243,7 +243,7 @@ class EditableMappingUpdater( } private def updateAgglomerateGraph(agglomerateId: Long, graph: AgglomerateGraph): Unit = { - val key = editableMappingService.agglomerateGraphKey(editableMappingId, agglomerateId) + val key = agglomerateGraphKey(editableMappingId, agglomerateId) agglomerateToGraphBuffer.put(key, (graph, false)) } @@ -428,7 +428,7 @@ class EditableMappingUpdater( case (chunkKey, _, version) => if (version > revertAction.sourceVersion) { editableMappingService - .getSegmentToAgglomerateChunk(editableMappingId, chunkKey, Some(revertAction.sourceVersion)) + .getSegmentToAgglomerateChunk(chunkKey, Some(revertAction.sourceVersion)) .futureBox .map { case Full(chunkData) => segmentToAgglomerateBuffer.put(chunkKey, (chunkData.toMap, false)) @@ -444,20 +444,22 @@ class EditableMappingUpdater( _ <- Fox.serialCombined(agglomerateToGraphNewestStream) { case (graphKey, _, version) => if (version > revertAction.sourceVersion) { - // TODO agglomerate id from graph key - editableMappingService - .getAgglomerateGraphForId(editableMappingId, - 0L, - remoteFallbackLayer, - userToken, - Some(revertAction.sourceVersion)) - .futureBox - .map { - case Full(graphData) => agglomerateToGraphBuffer.put(graphKey, (graphData, false)) - case Empty => agglomerateToGraphBuffer.put(graphKey, (emptyAgglomerateGraph, true)) - case Failure(msg, _, chain) => - Fox.failure(msg, Empty, chain) - } + for { + agglomerateId <- agglomerateIdFromAgglomerateGraphKey(graphKey) + _ <- editableMappingService + .getAgglomerateGraphForId(editableMappingId, + agglomerateId, + remoteFallbackLayer, + userToken, + Some(revertAction.sourceVersion)) + .futureBox + .map { + case Full(graphData) => agglomerateToGraphBuffer.put(graphKey, (graphData, false)) + case Empty => agglomerateToGraphBuffer.put(graphKey, (emptyAgglomerateGraph, true)) + case Failure(msg, _, chain) => + Fox.failure(msg, Empty, chain) + } + } yield () } else Fox.successful(()) } } yield oldInfo From 1325cac2b0ad03abd252422f87cc25aeaefc3f06 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 9 Jul 2024 13:47:54 +0200 Subject: [PATCH 012/150] fix version check, fix json serialization of update action --- .../tracings/editablemapping/EditableMappingService.scala | 2 +- .../editablemapping/EditableMappingUpdateActions.scala | 7 +++++++ .../tracings/editablemapping/EditableMappingUpdater.scala | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index 39fe85eb066..ff66dd20ac2 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -340,7 +340,7 @@ class EditableMappingService @Inject()( Some(batchFrom) )(fromJsonBytes[List[EditableMappingUpdateAction]]) } yield res - } + } ?~> "Failed to fetch editable mapping update actions from fossilDB" flat = updateActionBatches.flatten } yield flat } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala index 0bde8f4473b..96f01b912da 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala @@ -48,6 +48,10 @@ case class RevertToVersionUpdateAction(sourceVersion: Long, actionTimestamp: Opt override def addTimestamp(timestamp: Long): EditableMappingUpdateAction = this.copy(actionTimestamp = Some(timestamp)) } +object RevertToVersionUpdateAction { + implicit val jsonFormat: OFormat[RevertToVersionUpdateAction] = Json.format[RevertToVersionUpdateAction] +} + object EditableMappingUpdateAction { implicit object editableMappingUpdateActionFormat extends Format[EditableMappingUpdateAction] { @@ -55,6 +59,7 @@ object EditableMappingUpdateAction { (json \ "name").validate[String].flatMap { case "mergeAgglomerate" => (json \ "value").validate[MergeAgglomerateUpdateAction] case "splitAgglomerate" => (json \ "value").validate[SplitAgglomerateUpdateAction] + case "revertToVersion" => (json \ "value").validate[RevertToVersionUpdateAction] case unknownAction: String => JsError(s"Invalid update action s'$unknownAction'") } @@ -63,6 +68,8 @@ object EditableMappingUpdateAction { Json.obj("name" -> "splitAgglomerate", "value" -> Json.toJson(s)(SplitAgglomerateUpdateAction.jsonFormat)) case s: MergeAgglomerateUpdateAction => Json.obj("name" -> "mergeAgglomerate", "value" -> Json.toJson(s)(MergeAgglomerateUpdateAction.jsonFormat)) + case s: RevertToVersionUpdateAction => + Json.obj("name" -> "revertToVersion", "value" -> Json.toJson(s)(RevertToVersionUpdateAction.jsonFormat)) } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index 55b97b96e8d..89303d32040 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -414,7 +414,7 @@ class EditableMappingUpdater( private def revertToVersion(revertAction: RevertToVersionUpdateAction)( implicit ec: ExecutionContext): Fox[EditableMappingInfo] = for { - _ <- bool2Fox(revertAction.sourceVersion >= oldVersion) ?~> "trying to revert editable mapping to a version not yet present in the database" + _ <- bool2Fox(revertAction.sourceVersion <= oldVersion) ?~> "trying to revert editable mapping to a version not yet present in the database" oldInfo <- editableMappingService.getInfo(editableMappingId, Some(revertAction.sourceVersion), remoteFallbackLayer, From 2e62832b3a827d1aa65a4da0525740d23cbf4a3f Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 9 Jul 2024 14:05:33 +0200 Subject: [PATCH 013/150] wip update route --- fossildb/run.sh | 3 +- .../services/AccessTokenService.scala | 6 ++- .../controllers/DSAnnotationController.scala | 39 ++++++++++++++++++- ...alableminds.webknossos.tracingstore.routes | 1 + 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/fossildb/run.sh b/fossildb/run.sh index 39a622d2ee8..53f56832c2f 100755 --- a/fossildb/run.sh +++ b/fossildb/run.sh @@ -14,7 +14,6 @@ if [ ! -f "$JAR" ] || [ ! "$CURRENT_VERSION" == "$VERSION" ]; then wget -q --show-progress -O "$JAR" "$URL" fi -# Note that the editableMappings column is no longer used by wk. Still here for backwards compatibility. -COLLECTIONS="skeletons,skeletonUpdates,volumes,volumeData,volumeUpdates,volumeSegmentIndex,editableMappings,editableMappingUpdates,editableMappingsInfo,editableMappingsAgglomerateToGraph,editableMappingsSegmentToAgglomerate,annotations,annotationUpdates" +COLLECTIONS="skeletons,volumes,volumeData,volumeSegmentIndex,editableMappingsInfo,editableMappingsAgglomerateToGraph,editableMappingsSegmentToAgglomerate,annotations,annotationUpdates" exec java -jar "$JAR" -c "$COLLECTIONS" -d "$FOSSILDB_HOME/data" -b "$FOSSILDB_HOME/backup" diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AccessTokenService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AccessTokenService.scala index 4ba7159c4e3..b21c415b13c 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AccessTokenService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AccessTokenService.scala @@ -19,7 +19,7 @@ object AccessMode extends ExtendedEnumeration { object AccessResourceType extends ExtendedEnumeration { type AccessResourceType = Value - val datasource, tracing, webknossos, jobExport = Value + val datasource, tracing, annotation, webknossos, jobExport = Value } case class UserAccessAnswer(granted: Boolean, msg: Option[String] = None) @@ -42,9 +42,13 @@ object UserAccessRequest { def readTracing(tracingId: String): UserAccessRequest = UserAccessRequest(DataSourceId(tracingId, ""), AccessResourceType.tracing, AccessMode.read) + def writeTracing(tracingId: String): UserAccessRequest = UserAccessRequest(DataSourceId(tracingId, ""), AccessResourceType.tracing, AccessMode.write) + def writeAnnotation(annotationId: String): UserAccessRequest = + UserAccessRequest(DataSourceId(annotationId, ""), AccessResourceType.annotation, AccessMode.write) + def downloadJobExport(jobId: String): UserAccessRequest = UserAccessRequest(DataSourceId(jobId, ""), AccessResourceType.jobExport, AccessMode.read) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala index e534925f6c0..e4a5d8093ae 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala @@ -1,17 +1,26 @@ package com.scalableminds.webknossos.tracingstore.controllers import com.google.inject.Inject +import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.Annotation.AnnotationProto import com.scalableminds.webknossos.datastore.controllers.Controller import com.scalableminds.webknossos.datastore.services.UserAccessRequest -import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} +import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore, UpdateActionGroup} import com.scalableminds.webknossos.tracingstore.TracingStoreAccessTokenService +import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService +import play.api.libs.json.{Json, OFormat} import play.api.mvc.{Action, AnyContent, PlayBodyParsers} import scala.concurrent.{ExecutionContext, Future} +case class GenericUpdateActionGroup(transactionGroupCount: Int) +object GenericUpdateActionGroup { + implicit val jsonFormat: OFormat[GenericUpdateActionGroup] = Json.format[GenericUpdateActionGroup] +} + class DSAnnotationController @Inject()( accessTokenService: TracingStoreAccessTokenService, + slackNotificationService: TSSlackNotificationService, tracingDataStore: TracingDataStore)(implicit ec: ExecutionContext, bodyParsers: PlayBodyParsers) extends Controller with KeyValueStoreImplicits { @@ -26,6 +35,34 @@ class DSAnnotationController @Inject()( } } } + + def update(annotationId: String, token: Option[String]): Action[List[GenericUpdateActionGroup]] = + Action.async(validateJson[List[GenericUpdateActionGroup]]) { implicit request => + log() { + logTime(slackNotificationService.noticeSlowRequest) { + accessTokenService.validateAccess(UserAccessRequest.writeAnnotation(annotationId), + urlOrHeaderToken(token, request)) { + val updateGroups = request.body + if (updateGroups.forall(_.transactionGroupCount == 1)) { + //commitUpdates(tracingId, updateGroups, urlOrHeaderToken(token, request)).map(_ => Ok) + Fox.successful(Ok) + } else { + /*updateGroups + .foldLeft(tracingService.currentVersion(tracingId)) { (currentCommittedVersionFox, updateGroup) => + handleUpdateGroupForTransaction(tracingId, + currentCommittedVersionFox, + updateGroup, + urlOrHeaderToken(token, request)) + } + .map(_ => Ok) + + */ + Fox.successful(Ok) + } + } + } + } + } } // get version history diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 566242f4c52..da0b1f93131 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -6,6 +6,7 @@ GET /health @com.scalableminds.webknossos.tracingstore.controllers.Application.health POST /annotation/initialize @com.scalableminds.webknossos.tracingstore.controllers.DSAnnotationController.initialize(annotationId: String, token: Option[String]) +POST /annotation/update @com.scalableminds.webknossos.tracingstore.controllers.DSAnnotationController.update(annotationId: String, token: Option[String]) # Volume tracings POST /volume/save @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.save(token: Option[String]) From 950ab983ba7bbee6a4103afcb2043dc5175b6dc6 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 9 Jul 2024 14:54:15 +0200 Subject: [PATCH 014/150] annotationUpdates --- .../controllers/DSAnnotationController.scala | 10 +++++++++- .../tracingstore/tracings/TracingDataStore.scala | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala index e4a5d8093ae..a2f61ca8729 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala @@ -44,7 +44,7 @@ class DSAnnotationController @Inject()( urlOrHeaderToken(token, request)) { val updateGroups = request.body if (updateGroups.forall(_.transactionGroupCount == 1)) { - //commitUpdates(tracingId, updateGroups, urlOrHeaderToken(token, request)).map(_ => Ok) + commitUpdates(annotationId, updateGroups, urlOrHeaderToken(token, request)).map(_ => Ok) Fox.successful(Ok) } else { /*updateGroups @@ -63,6 +63,14 @@ class DSAnnotationController @Inject()( } } } + + private def commitUpdates(annotationId: String, + updateGroups: List[GenericUpdateActionGroup], + token: Option[String]): Fox[Unit] = { + val currentCommittedVersion: Fox[Long] = + tracingDataStore.annotationUpdates.getVersion(annotationId, mayBeEmpty = Some(true), emptyFallback = Some(0L)) + Fox.successful(()) + } } // get version history diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala index 629d79d18a9..4de87d378b3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala @@ -22,6 +22,8 @@ class TracingDataStore @Inject()(config: TracingStoreConfig, lazy val skeletons = new FossilDBClient("skeletons", config, slackNotificationService) + lazy val annotationUpdates = new FossilDBClient("annotationUpdates", config, slackNotificationService) + lazy val skeletonUpdates = new FossilDBClient("skeletonUpdates", config, slackNotificationService) lazy val volumes = new FossilDBClient("volumes", config, slackNotificationService) From 64e4e94d14f6a6728013da66f73c6bf088a13501 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 15 Jul 2024 12:01:39 +0200 Subject: [PATCH 015/150] WIP: AnnotationTransactionService --- .../tracingstore/TracingStoreModule.scala | 3 + .../AnnotationTransactionService.scala | 156 ++++++++++++++++++ .../annotation/DSAnnotationService.scala | 6 + .../controllers/DSAnnotationController.scala | 26 ++- 4 files changed, 182 insertions(+), 9 deletions(-) create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreModule.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreModule.scala index cd6fb91fc9d..c0d36e2265c 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreModule.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreModule.scala @@ -4,6 +4,7 @@ import org.apache.pekko.actor.ActorSystem import com.google.inject.AbstractModule import com.google.inject.name.Names import com.scalableminds.webknossos.datastore.services.AdHocMeshServiceHolder +import com.scalableminds.webknossos.tracingstore.annotation.AnnotationTransactionService import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService import com.scalableminds.webknossos.tracingstore.tracings.TracingDataStore import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService @@ -25,5 +26,7 @@ class TracingStoreModule extends AbstractModule { bind(classOf[EditableMappingService]).asEagerSingleton() bind(classOf[TSSlackNotificationService]).asEagerSingleton() bind(classOf[AdHocMeshServiceHolder]).asEagerSingleton() + bind(classOf[AnnotationTransactionService]).asEagerSingleton() } + } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala new file mode 100644 index 00000000000..9dc54bc07e3 --- /dev/null +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala @@ -0,0 +1,156 @@ +package com.scalableminds.webknossos.tracingstore.annotation + +import com.scalableminds.util.tools.{Fox, JsonHelper} +import com.scalableminds.util.tools.Fox.bool2Fox +import com.scalableminds.webknossos.tracingstore.TracingStoreRedisStore +import com.scalableminds.webknossos.tracingstore.controllers.GenericUpdateActionGroup +import play.api.http.Status.CONFLICT +import play.api.libs.json.Json + +import javax.inject.Inject +import scala.concurrent.ExecutionContext +import scala.concurrent.duration._ + +class AnnotationTransactionService @Inject()( + handledGroupIdStore: TracingStoreRedisStore, // TODO: instantiate here rather than with injection, give fix namespace prefix? + uncommittedUpdatesStore: TracingStoreRedisStore, + annotationService: DSAnnotationService) { + + private val transactionGroupExpiry: FiniteDuration = 24 hours + private val handledGroupCacheExpiry: FiniteDuration = 24 hours + + private def transactionGroupKey(annotationId: String, + transactionId: String, + transactionGroupIndex: Int, + version: Long) = + s"transactionGroup___${annotationId}___${transactionId}___${transactionGroupIndex}___$version" + + private def handledGroupKey(annotationId: String, transactionId: String, version: Long, transactionGroupIndex: Int) = + s"handledGroup___${annotationId}___${transactionId}___${version}___$transactionGroupIndex" + + private def patternFor(annotationId: String, transactionId: String) = + s"transactionGroup___${annotationId}___${transactionId}___*" + + def saveUncommitted(annotationId: String, + transactionId: String, + transactionGroupIndex: Int, + version: Long, + updateGroup: GenericUpdateActionGroup, + expiry: FiniteDuration)(implicit ec: ExecutionContext): Fox[Unit] = + for { + _ <- Fox.runIf(transactionGroupIndex > 0)( + Fox.assertTrue( + uncommittedUpdatesStore.contains(transactionGroupKey( + annotationId, + transactionId, + transactionGroupIndex - 1, + version))) ?~> s"Incorrect transaction index. Got: $transactionGroupIndex but ${transactionGroupIndex - 1} does not exist" ~> CONFLICT) + _ <- uncommittedUpdatesStore.insert( + transactionGroupKey(annotationId, transactionId, transactionGroupIndex, version), + Json.toJson(updateGroup).toString(), + Some(expiry)) + } yield () + + def handleUpdateGroupForTransaction(annotationId: String, + previousVersionFox: Fox[Long], + updateGroup: GenericUpdateActionGroup, + userToken: Option[String])(implicit ec: ExecutionContext): Fox[Long] = + for { + previousCommittedVersion: Long <- previousVersionFox + result <- if (previousCommittedVersion + 1 == updateGroup.version) { + if (updateGroup.transactionGroupCount == updateGroup.transactionGroupIndex + 1) { + // Received the last group of this transaction + commitWithPending(annotationId, updateGroup, userToken) + } else { + for { + _ <- saveUncommitted(annotationId, + updateGroup.transactionId, + updateGroup.transactionGroupIndex, + updateGroup.version, + updateGroup, + transactionGroupExpiry) + _ <- saveToHandledGroupIdStore(annotationId, + updateGroup.transactionId, + updateGroup.version, + updateGroup.transactionGroupIndex) + } yield previousCommittedVersion // no updates have been committed, do not yield version increase + } + } else { + failUnlessAlreadyHandled(updateGroup, annotationId, previousCommittedVersion) + } + } yield result + + // For an update group (that is the last of a transaction), fetch all previous uncommitted for the same transaction + // and commit them all. + private def commitWithPending(annotationId: String, updateGroup: GenericUpdateActionGroup, userToken: Option[String])( + implicit ec: ExecutionContext): Fox[Long] = + for { + previousActionGroupsToCommit <- getAllUncommittedFor(annotationId, updateGroup.transactionId) + _ <- bool2Fox( + previousActionGroupsToCommit + .exists(_.transactionGroupIndex == 0) || updateGroup.transactionGroupCount == 1) ?~> s"Trying to commit a transaction without a group that has transactionGroupIndex 0." + concatenatedGroup = concatenateUpdateGroupsOfTransaction(previousActionGroupsToCommit, updateGroup) + commitResult <- annotationService.commitUpdates(annotationId, List(concatenatedGroup), userToken) + _ <- removeAllUncommittedFor(annotationId, updateGroup.transactionId) + } yield commitResult + + private def removeAllUncommittedFor(tracingId: String, transactionId: String): Fox[Unit] = + uncommittedUpdatesStore.removeAllConditional(patternFor(tracingId, transactionId)) + + private def getAllUncommittedFor(annotationId: String, transactionId: String): Fox[List[GenericUpdateActionGroup]] = + for { + raw: Seq[String] <- uncommittedUpdatesStore.findAllConditional(patternFor(annotationId, transactionId)) + parsed: Seq[GenericUpdateActionGroup] = raw.flatMap(itemAsString => + JsonHelper.jsResultToOpt(Json.parse(itemAsString).validate[GenericUpdateActionGroup])) + } yield parsed.toList.sortBy(_.transactionGroupIndex) + + private def saveToHandledGroupIdStore(annotationId: String, + transactionId: String, + version: Long, + transactionGroupIndex: Int): Fox[Unit] = { + val key = handledGroupKey(annotationId, transactionId, version, transactionGroupIndex) + handledGroupIdStore.insert(key, "()", Some(handledGroupCacheExpiry)) + } + + private def handledGroupIdStoreContains(annotationId: String, + transactionId: String, + version: Long, + transactionGroupIndex: Int): Fox[Boolean] = + handledGroupIdStore.contains(handledGroupKey(annotationId, transactionId, version, transactionGroupIndex)) + + private def concatenateUpdateGroupsOfTransaction( + previousActionGroups: List[GenericUpdateActionGroup], + lastActionGroup: GenericUpdateActionGroup): GenericUpdateActionGroup = + if (previousActionGroups.isEmpty) lastActionGroup + else { + val allActionGroups = previousActionGroups :+ lastActionGroup + GenericUpdateActionGroup( + version = lastActionGroup.version, + timestamp = lastActionGroup.timestamp, + authorId = lastActionGroup.authorId, + actions = allActionGroups.flatMap(_.actions), + stats = lastActionGroup.stats, // the latest stats do count + info = lastActionGroup.info, // frontend sets this identically for all groups of transaction + transactionId = f"${lastActionGroup.transactionId}-concatenated", + transactionGroupCount = 1, + transactionGroupIndex = 0, + ) + } + + /* If this update group has already been “handled” (successfully saved as either committed or uncommitted), + * ignore it silently. This is in case the frontend sends a retry if it believes a save to be unsuccessful + * despite the backend receiving it just fine. + */ + private def failUnlessAlreadyHandled(updateGroup: GenericUpdateActionGroup, tracingId: String, previousVersion: Long)( + implicit ec: ExecutionContext): Fox[Long] = { + val errorMessage = s"Incorrect version. Expected: ${previousVersion + 1}; Got: ${updateGroup.version}" + for { + _ <- Fox.assertTrue( + handledGroupIdStoreContains(tracingId, + updateGroup.transactionId, + updateGroup.version, + updateGroup.transactionGroupIndex)) ?~> errorMessage ~> CONFLICT + } yield updateGroup.version + } + +} diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala index f6dffa0c274..83645cc7e3f 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala @@ -9,6 +9,8 @@ import com.scalableminds.webknossos.datastore.Annotation.{ UpdateLayerMetadataAnnotationUpdateAction, UpdateMetadataAnnotationUpdateAction } +import com.scalableminds.webknossos.tracingstore.controllers.GenericUpdateActionGroup +import com.scalableminds.webknossos.tracingstore.tracings.UpdateActionGroup import scalapb.GeneratedMessage import javax.inject.Inject @@ -17,6 +19,10 @@ import scala.concurrent.ExecutionContext class DSAnnotationService @Inject()() { def storeUpdate(updateAction: GeneratedMessage)(implicit ec: ExecutionContext): Fox[Unit] = Fox.successful(()) + def commitUpdates(tracingId: String, + updateGroups: List[GenericUpdateActionGroup], + userToken: Option[String]): Fox[Long] = ??? + def newestMaterializableVersion(annotationId: String): Fox[Long] = ??? def applyUpdate(annotation: AnnotationProto, updateAction: GeneratedMessage)( diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala index a2f61ca8729..c795f3ad532 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala @@ -7,13 +7,18 @@ import com.scalableminds.webknossos.datastore.controllers.Controller import com.scalableminds.webknossos.datastore.services.UserAccessRequest import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore, UpdateActionGroup} import com.scalableminds.webknossos.tracingstore.TracingStoreAccessTokenService +import com.scalableminds.webknossos.tracingstore.annotation.{AnnotationTransactionService, DSAnnotationService} import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService import play.api.libs.json.{Json, OFormat} import play.api.mvc.{Action, AnyContent, PlayBodyParsers} +import scala.concurrent.duration._ import scala.concurrent.{ExecutionContext, Future} -case class GenericUpdateActionGroup(transactionGroupCount: Int) +case class GenericUpdateActionGroup(transactionGroupCount: Int, + transactionGroupIndex: Int, + version: Long, + transactionId: String) object GenericUpdateActionGroup { implicit val jsonFormat: OFormat[GenericUpdateActionGroup] = Json.format[GenericUpdateActionGroup] } @@ -21,6 +26,8 @@ object GenericUpdateActionGroup { class DSAnnotationController @Inject()( accessTokenService: TracingStoreAccessTokenService, slackNotificationService: TSSlackNotificationService, + annotationService: DSAnnotationService, + transactionService: AnnotationTransactionService, tracingDataStore: TracingDataStore)(implicit ec: ExecutionContext, bodyParsers: PlayBodyParsers) extends Controller with KeyValueStoreImplicits { @@ -47,16 +54,15 @@ class DSAnnotationController @Inject()( commitUpdates(annotationId, updateGroups, urlOrHeaderToken(token, request)).map(_ => Ok) Fox.successful(Ok) } else { - /*updateGroups - .foldLeft(tracingService.currentVersion(tracingId)) { (currentCommittedVersionFox, updateGroup) => - handleUpdateGroupForTransaction(tracingId, - currentCommittedVersionFox, - updateGroup, - urlOrHeaderToken(token, request)) + updateGroups + .foldLeft(annotationService.newestMaterializableVersion(annotationId)) { + (currentCommittedVersionFox, updateGroup) => + transactionService.handleUpdateGroupForTransaction(annotationId, + currentCommittedVersionFox, + updateGroup, + urlOrHeaderToken(token, request)) } .map(_ => Ok) - - */ Fox.successful(Ok) } } @@ -64,6 +70,8 @@ class DSAnnotationController @Inject()( } } + private val transactionGroupExpiry: FiniteDuration = 24 hours + private def commitUpdates(annotationId: String, updateGroups: List[GenericUpdateActionGroup], token: Option[String]): Fox[Unit] = { From 3b8e39ae38a4b87e23d640d7fb05c3d39ec62a17 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 15 Jul 2024 13:41:08 +0200 Subject: [PATCH 016/150] finalize transaction service --- .../AnnotationTransactionService.scala | 44 +++++++++++++--- .../annotation/DSAnnotationService.scala | 30 ++++++++--- .../annotation/UpdateActions.scala | 28 +++++++++++ .../controllers/DSAnnotationController.scala | 50 ++++--------------- 4 files changed, 98 insertions(+), 54 deletions(-) create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala index 9dc54bc07e3..319cd00feab 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala @@ -3,7 +3,6 @@ package com.scalableminds.webknossos.tracingstore.annotation import com.scalableminds.util.tools.{Fox, JsonHelper} import com.scalableminds.util.tools.Fox.bool2Fox import com.scalableminds.webknossos.tracingstore.TracingStoreRedisStore -import com.scalableminds.webknossos.tracingstore.controllers.GenericUpdateActionGroup import play.api.http.Status.CONFLICT import play.api.libs.json.Json @@ -51,10 +50,10 @@ class AnnotationTransactionService @Inject()( Some(expiry)) } yield () - def handleUpdateGroupForTransaction(annotationId: String, - previousVersionFox: Fox[Long], - updateGroup: GenericUpdateActionGroup, - userToken: Option[String])(implicit ec: ExecutionContext): Fox[Long] = + private def handleUpdateGroupForTransaction(annotationId: String, + previousVersionFox: Fox[Long], + updateGroup: GenericUpdateActionGroup, + userToken: Option[String])(implicit ec: ExecutionContext): Fox[Long] = for { previousCommittedVersion: Long <- previousVersionFox result <- if (previousCommittedVersion + 1 == updateGroup.version) { @@ -90,7 +89,7 @@ class AnnotationTransactionService @Inject()( previousActionGroupsToCommit .exists(_.transactionGroupIndex == 0) || updateGroup.transactionGroupCount == 1) ?~> s"Trying to commit a transaction without a group that has transactionGroupIndex 0." concatenatedGroup = concatenateUpdateGroupsOfTransaction(previousActionGroupsToCommit, updateGroup) - commitResult <- annotationService.commitUpdates(annotationId, List(concatenatedGroup), userToken) + commitResult <- commitUpdates(annotationId, List(concatenatedGroup), userToken) _ <- removeAllUncommittedFor(annotationId, updateGroup.transactionId) } yield commitResult @@ -137,6 +136,39 @@ class AnnotationTransactionService @Inject()( ) } + def handleUpdateGroups(annotationId: String, updateGroups: List[GenericUpdateActionGroup], userToken: Option[String])( + implicit ec: ExecutionContext): Fox[Long] = + if (updateGroups.forall(_.transactionGroupCount == 1)) { + commitUpdates(annotationId, updateGroups, userToken) + } else { + updateGroups.foldLeft(annotationService.currentVersion(annotationId)) { + (currentCommittedVersionFox, updateGroup) => + handleUpdateGroupForTransaction(annotationId, currentCommittedVersionFox, updateGroup, userToken) + } + } + + // Perform version check and commit the passed updates + private def commitUpdates(annotationId: String, + updateGroups: List[GenericUpdateActionGroup], + userToken: Option[String])(implicit ec: ExecutionContext): Fox[Long] = + for { + _ <- annotationService.reportUpdates(annotationId, updateGroups, userToken) + currentCommittedVersion: Fox[Long] = annotationService.currentVersion(annotationId) + newVersion <- updateGroups.foldLeft(currentCommittedVersion) { (previousVersion, updateGroup) => + previousVersion.flatMap { prevVersion: Long => + if (prevVersion + 1 == updateGroup.version) { + for { + _ <- annotationService.handleUpdateGroup(annotationId, updateGroup, prevVersion, userToken) + _ <- saveToHandledGroupIdStore(annotationId, + updateGroup.transactionId, + updateGroup.version, + updateGroup.transactionGroupIndex) + } yield updateGroup.version + } else failUnlessAlreadyHandled(updateGroup, annotationId, prevVersion) + } + } + } yield newVersion + /* If this update group has already been “handled” (successfully saved as either committed or uncommitted), * ignore it silently. This is in case the frontend sends a retry if it believes a save to be unsuccessful * despite the backend receiving it just fine. diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala index 83645cc7e3f..67f553a572c 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala @@ -1,5 +1,6 @@ package com.scalableminds.webknossos.tracingstore.annotation +import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.Annotation.{ AddLayerAnnotationUpdateAction, @@ -9,21 +10,36 @@ import com.scalableminds.webknossos.datastore.Annotation.{ UpdateLayerMetadataAnnotationUpdateAction, UpdateMetadataAnnotationUpdateAction } -import com.scalableminds.webknossos.tracingstore.controllers.GenericUpdateActionGroup -import com.scalableminds.webknossos.tracingstore.tracings.UpdateActionGroup +import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingUpdatesReport} import scalapb.GeneratedMessage import javax.inject.Inject import scala.concurrent.ExecutionContext -class DSAnnotationService @Inject()() { +class DSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosClient) { def storeUpdate(updateAction: GeneratedMessage)(implicit ec: ExecutionContext): Fox[Unit] = Fox.successful(()) - def commitUpdates(tracingId: String, + def reportUpdates(annotationId: String, updateGroups: List[GenericUpdateActionGroup], - userToken: Option[String]): Fox[Long] = ??? - - def newestMaterializableVersion(annotationId: String): Fox[Long] = ??? + userToken: Option[String]): Fox[Unit] = + for { + _ <- remoteWebknossosClient.reportTracingUpdates( + TracingUpdatesReport( + annotationId, + timestamps = updateGroups.map(g => Instant(g.timestamp)), + statistics = updateGroups.flatMap(_.stats).lastOption, + significantChangesCount = updateGroups.map(_.significantChangesCount).sum, + viewChangesCount = updateGroups.map(_.viewChangesCount).sum, + userToken + )) + } yield () + + def currentVersion(annotationId: String): Fox[Long] = ??? + + def handleUpdateGroup(annotationId: String, + updateGroup: GenericUpdateActionGroup, + previousVersion: Long, + userToken: Option[String]): Fox[Unit] = ??? def applyUpdate(annotation: AnnotationProto, updateAction: GeneratedMessage)( implicit ec: ExecutionContext): Fox[AnnotationProto] = diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala new file mode 100644 index 00000000000..b32e68c4f50 --- /dev/null +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala @@ -0,0 +1,28 @@ +package com.scalableminds.webknossos.tracingstore.annotation + +import play.api.libs.json.{JsObject, Json, OFormat} + +case class GenericUpdateAction(actionTimestamp: Option[Long] = None) + +object GenericUpdateAction { + implicit val jsonFormat: OFormat[GenericUpdateAction] = Json.format[GenericUpdateAction] +} + +case class GenericUpdateActionGroup(version: Long, + timestamp: Long, + authorId: Option[String], + actions: List[GenericUpdateAction], + stats: Option[JsObject], + info: Option[String], + transactionId: String, + transactionGroupCount: Int, + transactionGroupIndex: Int) { + + def significantChangesCount: Int = 1 // TODO + + def viewChangesCount: Int = 1 // TODO +} + +object GenericUpdateActionGroup { + implicit val jsonFormat: OFormat[GenericUpdateActionGroup] = Json.format[GenericUpdateActionGroup] +} diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala index c795f3ad532..eff6f83c303 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala @@ -1,33 +1,21 @@ package com.scalableminds.webknossos.tracingstore.controllers import com.google.inject.Inject -import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.Annotation.AnnotationProto import com.scalableminds.webknossos.datastore.controllers.Controller import com.scalableminds.webknossos.datastore.services.UserAccessRequest -import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore, UpdateActionGroup} +import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} import com.scalableminds.webknossos.tracingstore.TracingStoreAccessTokenService -import com.scalableminds.webknossos.tracingstore.annotation.{AnnotationTransactionService, DSAnnotationService} +import com.scalableminds.webknossos.tracingstore.annotation.{AnnotationTransactionService, GenericUpdateActionGroup} import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService -import play.api.libs.json.{Json, OFormat} import play.api.mvc.{Action, AnyContent, PlayBodyParsers} -import scala.concurrent.duration._ -import scala.concurrent.{ExecutionContext, Future} - -case class GenericUpdateActionGroup(transactionGroupCount: Int, - transactionGroupIndex: Int, - version: Long, - transactionId: String) -object GenericUpdateActionGroup { - implicit val jsonFormat: OFormat[GenericUpdateActionGroup] = Json.format[GenericUpdateActionGroup] -} +import scala.concurrent.ExecutionContext class DSAnnotationController @Inject()( accessTokenService: TracingStoreAccessTokenService, slackNotificationService: TSSlackNotificationService, - annotationService: DSAnnotationService, - transactionService: AnnotationTransactionService, + annotationTransactionService: AnnotationTransactionService, tracingDataStore: TracingDataStore)(implicit ec: ExecutionContext, bodyParsers: PlayBodyParsers) extends Controller with KeyValueStoreImplicits { @@ -49,36 +37,16 @@ class DSAnnotationController @Inject()( logTime(slackNotificationService.noticeSlowRequest) { accessTokenService.validateAccess(UserAccessRequest.writeAnnotation(annotationId), urlOrHeaderToken(token, request)) { - val updateGroups = request.body - if (updateGroups.forall(_.transactionGroupCount == 1)) { - commitUpdates(annotationId, updateGroups, urlOrHeaderToken(token, request)).map(_ => Ok) - Fox.successful(Ok) - } else { - updateGroups - .foldLeft(annotationService.newestMaterializableVersion(annotationId)) { - (currentCommittedVersionFox, updateGroup) => - transactionService.handleUpdateGroupForTransaction(annotationId, - currentCommittedVersionFox, - updateGroup, - urlOrHeaderToken(token, request)) - } - .map(_ => Ok) - Fox.successful(Ok) - } + for { + _ <- annotationTransactionService.handleUpdateGroups(annotationId, + request.body, + urlOrHeaderToken(token, request)) + } yield Ok } } } } - private val transactionGroupExpiry: FiniteDuration = 24 hours - - private def commitUpdates(annotationId: String, - updateGroups: List[GenericUpdateActionGroup], - token: Option[String]): Fox[Unit] = { - val currentCommittedVersion: Fox[Long] = - tracingDataStore.annotationUpdates.getVersion(annotationId, mayBeEmpty = Some(true), emptyFallback = Some(0L)) - Fox.successful(()) - } } // get version history From 615d78c8e6d2223cea09f54ec88cc5e70c517763 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 15 Jul 2024 17:13:03 +0200 Subject: [PATCH 017/150] move skeleton updates to GenericUpdateAction trait --- .../annotation/UpdateActions.scala | 10 +- .../updating/SkeletonUpdateActions.scala | 275 ++++++++++-------- 2 files changed, 165 insertions(+), 120 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala index b32e68c4f50..062cc51d99e 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala @@ -2,7 +2,15 @@ package com.scalableminds.webknossos.tracingstore.annotation import play.api.libs.json.{JsObject, Json, OFormat} -case class GenericUpdateAction(actionTimestamp: Option[Long] = None) +trait GenericUpdateAction { + def actionTimestamp: Option[Long] + + def addTimestamp(timestamp: Long): GenericUpdateAction + + def addInfo(info: Option[String]): GenericUpdateAction + + def addAuthorId(authorId: Option[String]): GenericUpdateAction +} object GenericUpdateAction { implicit val jsonFormat: OFormat[GenericUpdateAction] = Json.format[GenericUpdateAction] diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala index 1dd5f64d5d9..8a0fb0c75e6 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala @@ -5,9 +5,12 @@ import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.util.geometry.{Vec3Double, Vec3Int} import com.scalableminds.webknossos.datastore.helpers.{NodeDefaults, ProtoGeometryImplicits} import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate +import com.scalableminds.webknossos.tracingstore.annotation.GenericUpdateAction import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.TreeType.TreeType import play.api.libs.json._ +trait SkeletonUpdateAction extends GenericUpdateAction + case class CreateTreeSkeletonAction(id: Int, color: Option[com.scalableminds.util.image.Color], name: String, @@ -21,9 +24,9 @@ case class CreateTreeSkeletonAction(id: Int, info: Option[String] = None, `type`: Option[TreeType] = None, edgesAreVisible: Option[Boolean]) - extends UpdateAction.SkeletonUpdateAction + extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { val newTree = Tree( id, Nil, @@ -39,12 +42,12 @@ case class CreateTreeSkeletonAction(id: Int, edgesAreVisible ) tracing.withTrees(newTree +: tracing.trees) - } + }*/ - override def addTimestamp(timestamp: Long): UpdateAction[SkeletonTracing] = + override def addTimestamp(timestamp: Long): GenericUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction[SkeletonTracing] = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction[SkeletonTracing] = + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addAuthorId(authorId: Option[String]): GenericUpdateAction = this.copy(actionAuthorId = authorId) } @@ -52,14 +55,16 @@ case class DeleteTreeSkeletonAction(id: Int, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends UpdateAction.SkeletonUpdateAction { - override def applyOn(tracing: SkeletonTracing): SkeletonTracing = - tracing.withTrees(tracing.trees.filter(_.treeId != id)) + extends SkeletonUpdateAction { + /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = + tracing.withTrees(tracing.trees.filter(_.treeId != id))*/ - override def addTimestamp(timestamp: Long): UpdateAction[SkeletonTracing] = + override def addTimestamp(timestamp: Long): GenericUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction[SkeletonTracing] = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction[SkeletonTracing] = + + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + + override def addAuthorId(authorId: Option[String]): GenericUpdateAction = this.copy(actionAuthorId = authorId) } @@ -74,9 +79,9 @@ case class UpdateTreeSkeletonAction(id: Int, actionAuthorId: Option[String] = None, info: Option[String] = None, `type`: Option[TreeType] = None) - extends UpdateAction.SkeletonUpdateAction + extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { def treeTransform(tree: Tree) = tree.copy( color = colorOptToProto(color).orElse(tree.color), @@ -89,12 +94,12 @@ case class UpdateTreeSkeletonAction(id: Int, ) tracing.withTrees(mapTrees(tracing, id, treeTransform)) - } + }*/ - override def addTimestamp(timestamp: Long): UpdateAction[SkeletonTracing] = + override def addTimestamp(timestamp: Long): GenericUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction[SkeletonTracing] = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction[SkeletonTracing] = + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addAuthorId(authorId: Option[String]): GenericUpdateAction = this.copy(actionAuthorId = authorId) } @@ -103,8 +108,9 @@ case class MergeTreeSkeletonAction(sourceId: Int, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends UpdateAction.SkeletonUpdateAction + extends SkeletonUpdateAction with SkeletonUpdateActionHelper { + /* // only nodes and edges are merged here, // other properties are managed explicitly // by the frontend with extra actions @@ -117,12 +123,14 @@ case class MergeTreeSkeletonAction(sourceId: Int, } tracing.withTrees(mapTrees(tracing, targetId, treeTransform).filter(_.treeId != sourceId)) - } + }*/ - override def addTimestamp(timestamp: Long): UpdateAction[SkeletonTracing] = + override def addTimestamp(timestamp: Long): GenericUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction[SkeletonTracing] = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction[SkeletonTracing] = + + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + + override def addAuthorId(authorId: Option[String]): GenericUpdateAction = this.copy(actionAuthorId = authorId) } @@ -132,8 +140,10 @@ case class MoveTreeComponentSkeletonAction(nodeIds: List[Int], actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends UpdateAction.SkeletonUpdateAction + extends SkeletonUpdateAction with SkeletonUpdateActionHelper { + + /* // this should only move a whole component, // that is disjoint from the rest of the tree override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { @@ -156,11 +166,14 @@ case class MoveTreeComponentSkeletonAction(nodeIds: List[Int], tracing.withTrees(tracing.trees.map(selectTree)) } + */ - override def addTimestamp(timestamp: Long): UpdateAction[SkeletonTracing] = + override def addTimestamp(timestamp: Long): GenericUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction[SkeletonTracing] = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction[SkeletonTracing] = + + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + + override def addAuthorId(authorId: Option[String]): GenericUpdateAction = this.copy(actionAuthorId = authorId) } @@ -170,17 +183,19 @@ case class CreateEdgeSkeletonAction(source: Int, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends UpdateAction.SkeletonUpdateAction + extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { def treeTransform(tree: Tree) = tree.withEdges(Edge(source, target) +: tree.edges) tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) - } + }*/ - override def addTimestamp(timestamp: Long): UpdateAction[SkeletonTracing] = + override def addTimestamp(timestamp: Long): GenericUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction[SkeletonTracing] = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction[SkeletonTracing] = + + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + + override def addAuthorId(authorId: Option[String]): GenericUpdateAction = this.copy(actionAuthorId = authorId) } @@ -190,17 +205,19 @@ case class DeleteEdgeSkeletonAction(source: Int, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends UpdateAction.SkeletonUpdateAction + extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { def treeTransform(tree: Tree) = tree.copy(edges = tree.edges.filter(_ != Edge(source, target))) tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) - } + }*/ - override def addTimestamp(timestamp: Long): UpdateAction[SkeletonTracing] = + override def addTimestamp(timestamp: Long): GenericUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction[SkeletonTracing] = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction[SkeletonTracing] = + + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + + override def addAuthorId(authorId: Option[String]): GenericUpdateAction = this.copy(actionAuthorId = authorId) } @@ -218,10 +235,10 @@ case class CreateNodeSkeletonAction(id: Int, actionAuthorId: Option[String] = None, info: Option[String] = None, additionalCoordinates: Option[Seq[AdditionalCoordinate]] = None) - extends UpdateAction.SkeletonUpdateAction + extends SkeletonUpdateAction with SkeletonUpdateActionHelper with ProtoGeometryImplicits { - override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { val rotationOrDefault = rotation getOrElse NodeDefaults.rotation val newNode = Node( id, @@ -239,12 +256,14 @@ case class CreateNodeSkeletonAction(id: Int, def treeTransform(tree: Tree) = tree.withNodes(newNode +: tree.nodes) tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) - } + }*/ - override def addTimestamp(timestamp: Long): UpdateAction[SkeletonTracing] = + override def addTimestamp(timestamp: Long): GenericUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction[SkeletonTracing] = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction[SkeletonTracing] = + + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + + override def addAuthorId(authorId: Option[String]): GenericUpdateAction = this.copy(actionAuthorId = authorId) } @@ -262,10 +281,10 @@ case class UpdateNodeSkeletonAction(id: Int, actionAuthorId: Option[String] = None, info: Option[String] = None, additionalCoordinates: Option[Seq[AdditionalCoordinate]] = None) - extends UpdateAction.SkeletonUpdateAction + extends SkeletonUpdateAction with SkeletonUpdateActionHelper with ProtoGeometryImplicits { - override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + /* override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { val rotationOrDefault = rotation getOrElse NodeDefaults.rotation val newNode = Node( @@ -286,11 +305,14 @@ case class UpdateNodeSkeletonAction(id: Int, tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) } + */ - override def addTimestamp(timestamp: Long): UpdateAction[SkeletonTracing] = + override def addTimestamp(timestamp: Long): GenericUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction[SkeletonTracing] = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction[SkeletonTracing] = + + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + + override def addAuthorId(authorId: Option[String]): GenericUpdateAction = this.copy(actionAuthorId = authorId) } @@ -300,37 +322,41 @@ case class DeleteNodeSkeletonAction(nodeId: Int, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends UpdateAction.SkeletonUpdateAction + extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { def treeTransform(tree: Tree) = tree.withNodes(tree.nodes.filter(_.id != nodeId)) tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) - } + }*/ - override def addTimestamp(timestamp: Long): UpdateAction[SkeletonTracing] = + override def addTimestamp(timestamp: Long): GenericUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addAuthorId(authorId: Option[String]): UpdateAction[SkeletonTracing] = + + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + + override def addAuthorId(authorId: Option[String]): GenericUpdateAction = this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): UpdateAction[SkeletonTracing] = this.copy(info = info) } case class UpdateTreeGroupsSkeletonAction(treeGroups: List[UpdateActionTreeGroup], actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends UpdateAction.SkeletonUpdateAction + extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - override def applyOn(tracing: SkeletonTracing): SkeletonTracing = - tracing.withTreeGroups(treeGroups.map(convertTreeGroup)) + /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = + tracing.withTreeGroups(treeGroups.map(convertTreeGroup))*/ - override def addTimestamp(timestamp: Long): UpdateAction[SkeletonTracing] = + override def addTimestamp(timestamp: Long): GenericUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addAuthorId(authorId: Option[String]): UpdateAction[SkeletonTracing] = + + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + + override def addAuthorId(authorId: Option[String]): GenericUpdateAction = this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): UpdateAction[SkeletonTracing] = this.copy(info = info) } case class UpdateTracingSkeletonAction(activeNode: Option[Int], @@ -342,9 +368,9 @@ case class UpdateTracingSkeletonAction(activeNode: Option[Int], actionAuthorId: Option[String] = None, info: Option[String] = None, editPositionAdditionalCoordinates: Option[Seq[AdditionalCoordinate]] = None) - extends UpdateAction.SkeletonUpdateAction + extends SkeletonUpdateAction with ProtoGeometryImplicits { - override def applyOn(tracing: SkeletonTracing): SkeletonTracing = + /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = tracing.copy( editPosition = editPosition, editRotation = editRotation, @@ -352,28 +378,31 @@ case class UpdateTracingSkeletonAction(activeNode: Option[Int], userBoundingBox = userBoundingBox, activeNodeId = activeNode, editPositionAdditionalCoordinates = AdditionalCoordinate.toProto(editPositionAdditionalCoordinates) - ) + )*/ - override def addTimestamp(timestamp: Long): UpdateAction[SkeletonTracing] = + override def addTimestamp(timestamp: Long): GenericUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction[SkeletonTracing] = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction[SkeletonTracing] = + + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + + override def addAuthorId(authorId: Option[String]): GenericUpdateAction = this.copy(actionAuthorId = authorId) - override def isViewOnlyChange: Boolean = true } case class RevertToVersionAction(sourceVersion: Long, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends UpdateAction.SkeletonUpdateAction { - override def applyOn(tracing: SkeletonTracing): SkeletonTracing = - throw new Exception("RevertToVersionAction applied on unversioned tracing") + extends SkeletonUpdateAction { + /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = + throw new Exception("RevertToVersionAction applied on unversioned tracing")*/ - override def addTimestamp(timestamp: Long): UpdateAction[SkeletonTracing] = + override def addTimestamp(timestamp: Long): GenericUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction[SkeletonTracing] = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction[SkeletonTracing] = + + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + + override def addAuthorId(authorId: Option[String]): GenericUpdateAction = this.copy(actionAuthorId = authorId) } @@ -382,20 +411,21 @@ case class UpdateTreeVisibility(treeId: Int, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends UpdateAction.SkeletonUpdateAction + extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { def treeTransform(tree: Tree) = tree.copy(isVisible = Some(isVisible)) tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) - } + }*/ - override def addTimestamp(timestamp: Long): UpdateAction[SkeletonTracing] = + override def addTimestamp(timestamp: Long): GenericUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction[SkeletonTracing] = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction[SkeletonTracing] = + + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + + override def addAuthorId(authorId: Option[String]): GenericUpdateAction = this.copy(actionAuthorId = authorId) - override def isViewOnlyChange: Boolean = true } case class UpdateTreeGroupVisibility(treeGroupId: Option[Int], @@ -403,9 +433,9 @@ case class UpdateTreeGroupVisibility(treeGroupId: Option[Int], actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends UpdateAction.SkeletonUpdateAction + extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { def updateTreeGroups(treeGroups: Seq[TreeGroup]) = { def treeTransform(tree: Tree) = if (treeGroups.exists(group => tree.groupId.contains(group.groupId))) @@ -425,14 +455,15 @@ case class UpdateTreeGroupVisibility(treeGroupId: Option[Int], .map(group => updateTreeGroups(GroupUtils.getAllChildrenTreeGroups(group))) .getOrElse(tracing) } - } + }*/ - override def addTimestamp(timestamp: Long): UpdateAction[SkeletonTracing] = + override def addTimestamp(timestamp: Long): GenericUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction[SkeletonTracing] = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction[SkeletonTracing] = + + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + + override def addAuthorId(authorId: Option[String]): GenericUpdateAction = this.copy(actionAuthorId = authorId) - override def isViewOnlyChange: Boolean = true } case class UpdateTreeEdgesVisibility(treeId: Int, @@ -440,34 +471,38 @@ case class UpdateTreeEdgesVisibility(treeId: Int, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends UpdateAction.SkeletonUpdateAction + extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + + /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { def treeTransform(tree: Tree) = tree.copy(edgesAreVisible = Some(edgesAreVisible)) tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) - } + }*/ - override def addTimestamp(timestamp: Long): UpdateAction[SkeletonTracing] = + override def addTimestamp(timestamp: Long): GenericUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction[SkeletonTracing] = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction[SkeletonTracing] = + + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + + override def addAuthorId(authorId: Option[String]): GenericUpdateAction = this.copy(actionAuthorId = authorId) - override def isViewOnlyChange: Boolean = true } case class UpdateUserBoundingBoxes(boundingBoxes: List[NamedBoundingBox], actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends UpdateAction.SkeletonUpdateAction { - override def applyOn(tracing: SkeletonTracing): SkeletonTracing = - tracing.withUserBoundingBoxes(boundingBoxes.map(_.toProto)) + extends SkeletonUpdateAction { + /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = + tracing.withUserBoundingBoxes(boundingBoxes.map(_.toProto))*/ - override def addTimestamp(timestamp: Long): UpdateAction[SkeletonTracing] = + override def addTimestamp(timestamp: Long): GenericUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction[SkeletonTracing] = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction[SkeletonTracing] = + + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + + override def addAuthorId(authorId: Option[String]): GenericUpdateAction = this.copy(actionAuthorId = authorId) } @@ -476,8 +511,8 @@ case class UpdateUserBoundingBoxVisibility(boundingBoxId: Option[Int], actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends UpdateAction.SkeletonUpdateAction { - override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + extends SkeletonUpdateAction { + /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { def updateUserBoundingBoxes() = tracing.userBoundingBoxes.map { boundingBox => if (boundingBoxId.forall(_ == boundingBox.id)) @@ -487,29 +522,31 @@ case class UpdateUserBoundingBoxVisibility(boundingBoxId: Option[Int], } tracing.withUserBoundingBoxes(updateUserBoundingBoxes()) - } + }*/ - override def addTimestamp(timestamp: Long): UpdateAction[SkeletonTracing] = + override def addTimestamp(timestamp: Long): GenericUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction[SkeletonTracing] = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction[SkeletonTracing] = + + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + + override def addAuthorId(authorId: Option[String]): GenericUpdateAction = this.copy(actionAuthorId = authorId) - override def isViewOnlyChange: Boolean = true } case class UpdateTdCamera(actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends UpdateAction.SkeletonUpdateAction { + extends SkeletonUpdateAction { - override def applyOn(tracing: SkeletonTracing): SkeletonTracing = tracing + /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = tracing*/ - override def addTimestamp(timestamp: Long): UpdateAction[SkeletonTracing] = + override def addTimestamp(timestamp: Long): GenericUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction[SkeletonTracing] = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction[SkeletonTracing] = + + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + + override def addAuthorId(authorId: Option[String]): GenericUpdateAction = this.copy(actionAuthorId = authorId) - override def isViewOnlyChange: Boolean = true } object CreateTreeSkeletonAction { @@ -568,10 +605,10 @@ object UpdateUserBoundingBoxVisibility { } object UpdateTdCamera { implicit val jsonFormat: OFormat[UpdateTdCamera] = Json.format[UpdateTdCamera] } -object SkeletonUpdateAction { +object GenericUpdateAction { - implicit object skeletonUpdateActionFormat extends Format[UpdateAction[SkeletonTracing]] { - override def reads(json: JsValue): JsResult[UpdateAction.SkeletonUpdateAction] = { + implicit object genericUpdateActionFormat extends Format[GenericUpdateAction] { + override def reads(json: JsValue): JsResult[GenericUpdateAction] = { val jsonValue = (json \ "value").as[JsObject] (json \ "name").as[String] match { case "createTree" => deserialize[CreateTreeSkeletonAction](jsonValue) @@ -605,7 +642,7 @@ object SkeletonUpdateAction { private val positionTransform = (JsPath \ "position").json.update(JsPath.read[List[Float]].map(position => Json.toJson(position.map(_.toInt)))) - override def writes(a: UpdateAction[SkeletonTracing]): JsObject = a match { + override def writes(a: GenericUpdateAction): JsObject = a match { case s: CreateTreeSkeletonAction => Json.obj("name" -> "createTree", "value" -> Json.toJson(s)(CreateTreeSkeletonAction.jsonFormat)) case s: DeleteTreeSkeletonAction => From 313d791c5f84c41a16a0bcd2ee11ce18564ad69d Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 16 Jul 2024 11:53:43 +0200 Subject: [PATCH 018/150] further move skeleton update actions --- .../annotation/DSAnnotationService.scala | 20 +- .../annotation/UpdateActions.scala | 108 +++++++++- .../skeleton/SkeletonTracingService.scala | 2 +- .../updating/SkeletonUpdateActions.scala | 188 ++++++------------ 4 files changed, 180 insertions(+), 138 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala index 67f553a572c..4813926c08e 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala @@ -10,13 +10,17 @@ import com.scalableminds.webknossos.datastore.Annotation.{ UpdateLayerMetadataAnnotationUpdateAction, UpdateMetadataAnnotationUpdateAction } +import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing +import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore, UpdateActionGroup} import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingUpdatesReport} import scalapb.GeneratedMessage import javax.inject.Inject import scala.concurrent.ExecutionContext -class DSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosClient) { +class DSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosClient, + tracingDataStore: TracingDataStore) + extends KeyValueStoreImplicits { def storeUpdate(updateAction: GeneratedMessage)(implicit ec: ExecutionContext): Fox[Unit] = Fox.successful(()) def reportUpdates(annotationId: String, @@ -37,9 +41,19 @@ class DSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl def currentVersion(annotationId: String): Fox[Long] = ??? def handleUpdateGroup(annotationId: String, - updateGroup: GenericUpdateActionGroup, + updateActionGroup: GenericUpdateActionGroup, previousVersion: Long, - userToken: Option[String]): Fox[Unit] = ??? + userToken: Option[String]): Fox[Unit] = + // TODO apply volume updates directly? transform to compact? + tracingDataStore.annotationUpdates.put( + annotationId, + updateActionGroup.version, + updateActionGroup.actions + .map(_.addTimestamp(updateActionGroup.timestamp).addAuthorId(updateActionGroup.authorId)) match { //to the first action in the group, attach the group's info + case Nil => List[GenericUpdateAction]() + case first :: rest => first.addInfo(updateActionGroup.info) :: rest + } + ) def applyUpdate(annotation: AnnotationProto, updateAction: GeneratedMessage)( implicit ec: ExecutionContext): Fox[AnnotationProto] = diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala index 062cc51d99e..b8dd3651321 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala @@ -1,6 +1,27 @@ package com.scalableminds.webknossos.tracingstore.annotation -import play.api.libs.json.{JsObject, Json, OFormat} +import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{ + CreateEdgeSkeletonAction, + CreateNodeSkeletonAction, + CreateTreeSkeletonAction, + DeleteEdgeSkeletonAction, + DeleteNodeSkeletonAction, + DeleteTreeSkeletonAction, + MergeTreeSkeletonAction, + MoveTreeComponentSkeletonAction, + RevertToVersionSkeletonAction, + UpdateNodeSkeletonAction, + UpdateTdCameraSkeletonAction, + UpdateTracingSkeletonAction, + UpdateTreeEdgesVisibilitySkeletonAction, + UpdateTreeGroupVisibilitySkeletonAction, + UpdateTreeGroupsSkeletonAction, + UpdateTreeSkeletonAction, + UpdateTreeVisibilitySkeletonAction, + UpdateUserBoundingBoxVisibilitySkeletonAction, + UpdateUserBoundingBoxesSkeletonAction +} +import play.api.libs.json.{Format, JsObject, JsPath, JsResult, JsValue, Json, OFormat, Reads} trait GenericUpdateAction { def actionTimestamp: Option[Long] @@ -13,7 +34,90 @@ trait GenericUpdateAction { } object GenericUpdateAction { - implicit val jsonFormat: OFormat[GenericUpdateAction] = Json.format[GenericUpdateAction] + + implicit object genericUpdateActionFormat extends Format[GenericUpdateAction] { + override def reads(json: JsValue): JsResult[GenericUpdateAction] = { + val jsonValue = (json \ "value").as[JsObject] + (json \ "name").as[String] match { + case "createTree" => deserialize[CreateTreeSkeletonAction](jsonValue) + case "deleteTree" => deserialize[DeleteTreeSkeletonAction](jsonValue) + case "updateTree" => deserialize[UpdateTreeSkeletonAction](jsonValue) + case "mergeTree" => deserialize[MergeTreeSkeletonAction](jsonValue) + case "moveTreeComponent" => deserialize[MoveTreeComponentSkeletonAction](jsonValue) + case "createNode" => deserialize[CreateNodeSkeletonAction](jsonValue, shouldTransformPositions = true) + case "deleteNode" => deserialize[DeleteNodeSkeletonAction](jsonValue) + case "updateNode" => deserialize[UpdateNodeSkeletonAction](jsonValue, shouldTransformPositions = true) + case "createEdge" => deserialize[CreateEdgeSkeletonAction](jsonValue) + case "deleteEdge" => deserialize[DeleteEdgeSkeletonAction](jsonValue) + case "updateTreeGroups" => deserialize[UpdateTreeGroupsSkeletonAction](jsonValue) + case "updateTracing" => deserialize[UpdateTracingSkeletonAction](jsonValue) + case "revertToVersion" => deserialize[RevertToVersionSkeletonAction](jsonValue) + case "updateTreeVisibility" => deserialize[UpdateTreeVisibilitySkeletonAction](jsonValue) + case "updateTreeGroupVisibility" => deserialize[UpdateTreeGroupVisibilitySkeletonAction](jsonValue) + case "updateTreeEdgesVisibility" => deserialize[UpdateTreeEdgesVisibilitySkeletonAction](jsonValue) + case "updateUserBoundingBoxes" => deserialize[UpdateUserBoundingBoxesSkeletonAction](jsonValue) + case "updateUserBoundingBoxVisibility" => + deserialize[UpdateUserBoundingBoxVisibilitySkeletonAction](jsonValue) + case "updateTdCamera" => deserialize[UpdateTdCameraSkeletonAction](jsonValue) + } + } + + private def deserialize[T](json: JsValue, shouldTransformPositions: Boolean = false)( + implicit tjs: Reads[T]): JsResult[T] = + if (shouldTransformPositions) + json.transform(positionTransform).get.validate[T] + else + json.validate[T] + + private val positionTransform = + (JsPath \ "position").json.update(JsPath.read[List[Float]].map(position => Json.toJson(position.map(_.toInt)))) + + override def writes(a: GenericUpdateAction): JsObject = a match { + case s: CreateTreeSkeletonAction => + Json.obj("name" -> "createTree", "value" -> Json.toJson(s)(CreateTreeSkeletonAction.jsonFormat)) + case s: DeleteTreeSkeletonAction => + Json.obj("name" -> "deleteTree", "value" -> Json.toJson(s)(DeleteTreeSkeletonAction.jsonFormat)) + case s: UpdateTreeSkeletonAction => + Json.obj("name" -> "updateTree", "value" -> Json.toJson(s)(UpdateTreeSkeletonAction.jsonFormat)) + case s: MergeTreeSkeletonAction => + Json.obj("name" -> "mergeTree", "value" -> Json.toJson(s)(MergeTreeSkeletonAction.jsonFormat)) + case s: MoveTreeComponentSkeletonAction => + Json.obj("name" -> "moveTreeComponent", "value" -> Json.toJson(s)(MoveTreeComponentSkeletonAction.jsonFormat)) + case s: CreateNodeSkeletonAction => + Json.obj("name" -> "createNode", "value" -> Json.toJson(s)(CreateNodeSkeletonAction.jsonFormat)) + case s: DeleteNodeSkeletonAction => + Json.obj("name" -> "deleteNode", "value" -> Json.toJson(s)(DeleteNodeSkeletonAction.jsonFormat)) + case s: UpdateNodeSkeletonAction => + Json.obj("name" -> "updateNode", "value" -> Json.toJson(s)(UpdateNodeSkeletonAction.jsonFormat)) + case s: CreateEdgeSkeletonAction => + Json.obj("name" -> "createEdge", "value" -> Json.toJson(s)(CreateEdgeSkeletonAction.jsonFormat)) + case s: DeleteEdgeSkeletonAction => + Json.obj("name" -> "deleteEdge", "value" -> Json.toJson(s)(DeleteEdgeSkeletonAction.jsonFormat)) + case s: UpdateTreeGroupsSkeletonAction => + Json.obj("name" -> "updateTreeGroups", "value" -> Json.toJson(s)(UpdateTreeGroupsSkeletonAction.jsonFormat)) + case s: UpdateTracingSkeletonAction => + Json.obj("name" -> "updateTracing", "value" -> Json.toJson(s)(UpdateTracingSkeletonAction.jsonFormat)) + case s: RevertToVersionSkeletonAction => + Json.obj("name" -> "revertToVersion", "value" -> Json.toJson(s)(RevertToVersionSkeletonAction.jsonFormat)) + case s: UpdateTreeVisibilitySkeletonAction => + Json.obj("name" -> "updateTreeVisibility", + "value" -> Json.toJson(s)(UpdateTreeVisibilitySkeletonAction.jsonFormat)) + case s: UpdateTreeGroupVisibilitySkeletonAction => + Json.obj("name" -> "updateTreeGroupVisibility", + "value" -> Json.toJson(s)(UpdateTreeGroupVisibilitySkeletonAction.jsonFormat)) + case s: UpdateTreeEdgesVisibilitySkeletonAction => + Json.obj("name" -> "updateTreeEdgesVisibility", + "value" -> Json.toJson(s)(UpdateTreeEdgesVisibilitySkeletonAction.jsonFormat)) + case s: UpdateUserBoundingBoxesSkeletonAction => + Json.obj("name" -> "updateUserBoundingBoxes", + "value" -> Json.toJson(s)(UpdateUserBoundingBoxesSkeletonAction.jsonFormat)) + case s: UpdateUserBoundingBoxVisibilitySkeletonAction => + Json.obj("name" -> "updateUserBoundingBoxVisibility", + "value" -> Json.toJson(s)(UpdateUserBoundingBoxVisibilitySkeletonAction.jsonFormat)) + case s: UpdateTdCameraSkeletonAction => + Json.obj("name" -> "updateTdCamera", "value" -> Json.toJson(s)(UpdateTdCameraSkeletonAction.jsonFormat)) + } + } } case class GenericUpdateActionGroup(version: Long, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala index 66154842b1d..5be705493fa 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala @@ -118,7 +118,7 @@ class SkeletonTracingService @Inject()( case Full(tracing) => remainingUpdates match { case List() => Fox.successful(tracing) - case RevertToVersionAction(sourceVersion, _, _, _) :: tail => + case RevertToVersionSkeletonAction(sourceVersion, _, _, _) :: tail => val sourceTracing = find(tracingId, Some(sourceVersion), useCache = false, applyUpdates = true) updateIter(sourceTracing, tail) case update :: tail => updateIter(Full(update.applyOn(tracing)), tail) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala index 8a0fb0c75e6..f449ddd32ec 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala @@ -1,9 +1,8 @@ package com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating -import com.scalableminds.webknossos.datastore.SkeletonTracing._ import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.util.geometry.{Vec3Double, Vec3Int} -import com.scalableminds.webknossos.datastore.helpers.{NodeDefaults, ProtoGeometryImplicits} +import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate import com.scalableminds.webknossos.tracingstore.annotation.GenericUpdateAction import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.TreeType.TreeType @@ -389,10 +388,10 @@ case class UpdateTracingSkeletonAction(activeNode: Option[Int], this.copy(actionAuthorId = authorId) } -case class RevertToVersionAction(sourceVersion: Long, - actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) +case class RevertToVersionSkeletonAction(sourceVersion: Long, + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) extends SkeletonUpdateAction { /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = throw new Exception("RevertToVersionAction applied on unversioned tracing")*/ @@ -406,11 +405,11 @@ case class RevertToVersionAction(sourceVersion: Long, this.copy(actionAuthorId = authorId) } -case class UpdateTreeVisibility(treeId: Int, - isVisible: Boolean, - actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) +case class UpdateTreeVisibilitySkeletonAction(treeId: Int, + isVisible: Boolean, + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) extends SkeletonUpdateAction with SkeletonUpdateActionHelper { /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { @@ -428,11 +427,11 @@ case class UpdateTreeVisibility(treeId: Int, this.copy(actionAuthorId = authorId) } -case class UpdateTreeGroupVisibility(treeGroupId: Option[Int], - isVisible: Boolean, - actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) +case class UpdateTreeGroupVisibilitySkeletonAction(treeGroupId: Option[Int], + isVisible: Boolean, + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) extends SkeletonUpdateAction with SkeletonUpdateActionHelper { /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { @@ -466,11 +465,11 @@ case class UpdateTreeGroupVisibility(treeGroupId: Option[Int], this.copy(actionAuthorId = authorId) } -case class UpdateTreeEdgesVisibility(treeId: Int, - edgesAreVisible: Boolean, - actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) +case class UpdateTreeEdgesVisibilitySkeletonAction(treeId: Int, + edgesAreVisible: Boolean, + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) extends SkeletonUpdateAction with SkeletonUpdateActionHelper { @@ -489,10 +488,10 @@ case class UpdateTreeEdgesVisibility(treeId: Int, this.copy(actionAuthorId = authorId) } -case class UpdateUserBoundingBoxes(boundingBoxes: List[NamedBoundingBox], - actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) +case class UpdateUserBoundingBoxesSkeletonAction(boundingBoxes: List[NamedBoundingBox], + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) extends SkeletonUpdateAction { /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = tracing.withUserBoundingBoxes(boundingBoxes.map(_.toProto))*/ @@ -506,11 +505,11 @@ case class UpdateUserBoundingBoxes(boundingBoxes: List[NamedBoundingBox], this.copy(actionAuthorId = authorId) } -case class UpdateUserBoundingBoxVisibility(boundingBoxId: Option[Int], - isVisible: Boolean, - actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) +case class UpdateUserBoundingBoxVisibilitySkeletonAction(boundingBoxId: Option[Int], + isVisible: Boolean, + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) extends SkeletonUpdateAction { /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { def updateUserBoundingBoxes() = @@ -533,9 +532,9 @@ case class UpdateUserBoundingBoxVisibility(boundingBoxId: Option[Int], this.copy(actionAuthorId = authorId) } -case class UpdateTdCamera(actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) +case class UpdateTdCameraSkeletonAction(actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) extends SkeletonUpdateAction { /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = tracing*/ @@ -585,103 +584,28 @@ object UpdateTreeGroupsSkeletonAction { object UpdateTracingSkeletonAction { implicit val jsonFormat: OFormat[UpdateTracingSkeletonAction] = Json.format[UpdateTracingSkeletonAction] } -object RevertToVersionAction { - implicit val jsonFormat: OFormat[RevertToVersionAction] = Json.format[RevertToVersionAction] -} -object UpdateTreeVisibility { - implicit val jsonFormat: OFormat[UpdateTreeVisibility] = Json.format[UpdateTreeVisibility] -} -object UpdateTreeGroupVisibility { - implicit val jsonFormat: OFormat[UpdateTreeGroupVisibility] = Json.format[UpdateTreeGroupVisibility] -} -object UpdateTreeEdgesVisibility { - implicit val jsonFormat: OFormat[UpdateTreeEdgesVisibility] = Json.format[UpdateTreeEdgesVisibility] -} -object UpdateUserBoundingBoxes { - implicit val jsonFormat: OFormat[UpdateUserBoundingBoxes] = Json.format[UpdateUserBoundingBoxes] -} -object UpdateUserBoundingBoxVisibility { - implicit val jsonFormat: OFormat[UpdateUserBoundingBoxVisibility] = Json.format[UpdateUserBoundingBoxVisibility] -} -object UpdateTdCamera { implicit val jsonFormat: OFormat[UpdateTdCamera] = Json.format[UpdateTdCamera] } - -object GenericUpdateAction { - - implicit object genericUpdateActionFormat extends Format[GenericUpdateAction] { - override def reads(json: JsValue): JsResult[GenericUpdateAction] = { - val jsonValue = (json \ "value").as[JsObject] - (json \ "name").as[String] match { - case "createTree" => deserialize[CreateTreeSkeletonAction](jsonValue) - case "deleteTree" => deserialize[DeleteTreeSkeletonAction](jsonValue) - case "updateTree" => deserialize[UpdateTreeSkeletonAction](jsonValue) - case "mergeTree" => deserialize[MergeTreeSkeletonAction](jsonValue) - case "moveTreeComponent" => deserialize[MoveTreeComponentSkeletonAction](jsonValue) - case "createNode" => deserialize[CreateNodeSkeletonAction](jsonValue, shouldTransformPositions = true) - case "deleteNode" => deserialize[DeleteNodeSkeletonAction](jsonValue) - case "updateNode" => deserialize[UpdateNodeSkeletonAction](jsonValue, shouldTransformPositions = true) - case "createEdge" => deserialize[CreateEdgeSkeletonAction](jsonValue) - case "deleteEdge" => deserialize[DeleteEdgeSkeletonAction](jsonValue) - case "updateTreeGroups" => deserialize[UpdateTreeGroupsSkeletonAction](jsonValue) - case "updateTracing" => deserialize[UpdateTracingSkeletonAction](jsonValue) - case "revertToVersion" => deserialize[RevertToVersionAction](jsonValue) - case "updateTreeVisibility" => deserialize[UpdateTreeVisibility](jsonValue) - case "updateTreeGroupVisibility" => deserialize[UpdateTreeGroupVisibility](jsonValue) - case "updateTreeEdgesVisibility" => deserialize[UpdateTreeEdgesVisibility](jsonValue) - case "updateUserBoundingBoxes" => deserialize[UpdateUserBoundingBoxes](jsonValue) - case "updateUserBoundingBoxVisibility" => deserialize[UpdateUserBoundingBoxVisibility](jsonValue) - case "updateTdCamera" => deserialize[UpdateTdCamera](jsonValue) - } - } - - def deserialize[T](json: JsValue, shouldTransformPositions: Boolean = false)(implicit tjs: Reads[T]): JsResult[T] = - if (shouldTransformPositions) - json.transform(positionTransform).get.validate[T] - else - json.validate[T] - - private val positionTransform = - (JsPath \ "position").json.update(JsPath.read[List[Float]].map(position => Json.toJson(position.map(_.toInt)))) - - override def writes(a: GenericUpdateAction): JsObject = a match { - case s: CreateTreeSkeletonAction => - Json.obj("name" -> "createTree", "value" -> Json.toJson(s)(CreateTreeSkeletonAction.jsonFormat)) - case s: DeleteTreeSkeletonAction => - Json.obj("name" -> "deleteTree", "value" -> Json.toJson(s)(DeleteTreeSkeletonAction.jsonFormat)) - case s: UpdateTreeSkeletonAction => - Json.obj("name" -> "updateTree", "value" -> Json.toJson(s)(UpdateTreeSkeletonAction.jsonFormat)) - case s: MergeTreeSkeletonAction => - Json.obj("name" -> "mergeTree", "value" -> Json.toJson(s)(MergeTreeSkeletonAction.jsonFormat)) - case s: MoveTreeComponentSkeletonAction => - Json.obj("name" -> "moveTreeComponent", "value" -> Json.toJson(s)(MoveTreeComponentSkeletonAction.jsonFormat)) - case s: CreateNodeSkeletonAction => - Json.obj("name" -> "createNode", "value" -> Json.toJson(s)(CreateNodeSkeletonAction.jsonFormat)) - case s: DeleteNodeSkeletonAction => - Json.obj("name" -> "deleteNode", "value" -> Json.toJson(s)(DeleteNodeSkeletonAction.jsonFormat)) - case s: UpdateNodeSkeletonAction => - Json.obj("name" -> "updateNode", "value" -> Json.toJson(s)(UpdateNodeSkeletonAction.jsonFormat)) - case s: CreateEdgeSkeletonAction => - Json.obj("name" -> "createEdge", "value" -> Json.toJson(s)(CreateEdgeSkeletonAction.jsonFormat)) - case s: DeleteEdgeSkeletonAction => - Json.obj("name" -> "deleteEdge", "value" -> Json.toJson(s)(DeleteEdgeSkeletonAction.jsonFormat)) - case s: UpdateTreeGroupsSkeletonAction => - Json.obj("name" -> "updateTreeGroups", "value" -> Json.toJson(s)(UpdateTreeGroupsSkeletonAction.jsonFormat)) - case s: UpdateTracingSkeletonAction => - Json.obj("name" -> "updateTracing", "value" -> Json.toJson(s)(UpdateTracingSkeletonAction.jsonFormat)) - case s: RevertToVersionAction => - Json.obj("name" -> "revertToVersion", "value" -> Json.toJson(s)(RevertToVersionAction.jsonFormat)) - case s: UpdateTreeVisibility => - Json.obj("name" -> "updateTreeVisibility", "value" -> Json.toJson(s)(UpdateTreeVisibility.jsonFormat)) - case s: UpdateTreeGroupVisibility => - Json.obj("name" -> "updateTreeGroupVisibility", "value" -> Json.toJson(s)(UpdateTreeGroupVisibility.jsonFormat)) - case s: UpdateTreeEdgesVisibility => - Json.obj("name" -> "updateTreeEdgesVisibility", "value" -> Json.toJson(s)(UpdateTreeEdgesVisibility.jsonFormat)) - case s: UpdateUserBoundingBoxes => - Json.obj("name" -> "updateUserBoundingBoxes", "value" -> Json.toJson(s)(UpdateUserBoundingBoxes.jsonFormat)) - case s: UpdateUserBoundingBoxVisibility => - Json.obj("name" -> "updateUserBoundingBoxVisibility", - "value" -> Json.toJson(s)(UpdateUserBoundingBoxVisibility.jsonFormat)) - case s: UpdateTdCamera => - Json.obj("name" -> "updateTdCamera", "value" -> Json.toJson(s)(UpdateTdCamera.jsonFormat)) - } - } +object RevertToVersionSkeletonAction { + implicit val jsonFormat: OFormat[RevertToVersionSkeletonAction] = Json.format[RevertToVersionSkeletonAction] +} +object UpdateTreeVisibilitySkeletonAction { + implicit val jsonFormat: OFormat[UpdateTreeVisibilitySkeletonAction] = Json.format[UpdateTreeVisibilitySkeletonAction] +} +object UpdateTreeGroupVisibilitySkeletonAction { + implicit val jsonFormat: OFormat[UpdateTreeGroupVisibilitySkeletonAction] = + Json.format[UpdateTreeGroupVisibilitySkeletonAction] +} +object UpdateTreeEdgesVisibilitySkeletonAction { + implicit val jsonFormat: OFormat[UpdateTreeEdgesVisibilitySkeletonAction] = + Json.format[UpdateTreeEdgesVisibilitySkeletonAction] +} +object UpdateUserBoundingBoxesSkeletonAction { + implicit val jsonFormat: OFormat[UpdateUserBoundingBoxesSkeletonAction] = + Json.format[UpdateUserBoundingBoxesSkeletonAction] +} +object UpdateUserBoundingBoxVisibilitySkeletonAction { + implicit val jsonFormat: OFormat[UpdateUserBoundingBoxVisibilitySkeletonAction] = + Json.format[UpdateUserBoundingBoxVisibilitySkeletonAction] +} +object UpdateTdCameraSkeletonAction { + implicit val jsonFormat: OFormat[UpdateTdCameraSkeletonAction] = Json.format[UpdateTdCameraSkeletonAction] } From b6a066df37f4209d290156670b88e6051f58ad0b Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 16 Jul 2024 13:47:02 +0200 Subject: [PATCH 019/150] json for volume update actions --- .../annotation/UpdateActions.scala | 40 +++- .../controllers/VolumeTracingController.scala | 4 +- .../tracingstore/tracings/UpdateActions.scala | 5 - .../tracings/volume/VolumeUpdateActions.scala | 175 ++++++++---------- 4 files changed, 114 insertions(+), 110 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala index b8dd3651321..212fd9c4c84 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala @@ -21,7 +21,23 @@ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{ UpdateUserBoundingBoxVisibilitySkeletonAction, UpdateUserBoundingBoxesSkeletonAction } -import play.api.libs.json.{Format, JsObject, JsPath, JsResult, JsValue, Json, OFormat, Reads} +import com.scalableminds.webknossos.tracingstore.tracings.volume.{ + CreateSegmentVolumeAction, + DeleteSegmentDataVolumeAction, + DeleteSegmentVolumeAction, + ImportVolumeData, + RemoveFallbackLayer, + RevertToVersionVolumeAction, + UpdateBucketVolumeAction, + UpdateMappingNameVolumeAction, + UpdateSegmentGroupsVolumeAction, + UpdateSegmentVolumeAction, + UpdateTdCamera, + UpdateTracingVolumeAction, + UpdateUserBoundingBoxVisibility, + UpdateUserBoundingBoxes +} +import play.api.libs.json.{Format, JsError, JsObject, JsPath, JsResult, JsValue, Json, OFormat, Reads} trait GenericUpdateAction { def actionTimestamp: Option[Long] @@ -31,6 +47,8 @@ trait GenericUpdateAction { def addInfo(info: Option[String]): GenericUpdateAction def addAuthorId(authorId: Option[String]): GenericUpdateAction + + def isViewOnlyChange: Boolean = false } object GenericUpdateAction { @@ -50,16 +68,28 @@ object GenericUpdateAction { case "createEdge" => deserialize[CreateEdgeSkeletonAction](jsonValue) case "deleteEdge" => deserialize[DeleteEdgeSkeletonAction](jsonValue) case "updateTreeGroups" => deserialize[UpdateTreeGroupsSkeletonAction](jsonValue) - case "updateTracing" => deserialize[UpdateTracingSkeletonAction](jsonValue) - case "revertToVersion" => deserialize[RevertToVersionSkeletonAction](jsonValue) + case "updateSkeletonTracing" => deserialize[UpdateTracingSkeletonAction](jsonValue) case "updateTreeVisibility" => deserialize[UpdateTreeVisibilitySkeletonAction](jsonValue) case "updateTreeGroupVisibility" => deserialize[UpdateTreeGroupVisibilitySkeletonAction](jsonValue) case "updateTreeEdgesVisibility" => deserialize[UpdateTreeEdgesVisibilitySkeletonAction](jsonValue) case "updateUserBoundingBoxes" => deserialize[UpdateUserBoundingBoxesSkeletonAction](jsonValue) case "updateUserBoundingBoxVisibility" => deserialize[UpdateUserBoundingBoxVisibilitySkeletonAction](jsonValue) - case "updateTdCamera" => deserialize[UpdateTdCameraSkeletonAction](jsonValue) - } + case "updateBucket" => deserialize[UpdateBucketVolumeAction](jsonValue) + case "updateVolumeTracing" => deserialize[UpdateTracingVolumeAction](jsonValue) + case "updateUserBoundingBoxes" => deserialize[UpdateUserBoundingBoxes](jsonValue) + case "updateUserBoundingBoxVisibility" => deserialize[UpdateUserBoundingBoxVisibility](jsonValue) + case "removeFallbackLayer" => deserialize[RemoveFallbackLayer](jsonValue) + case "importVolumeTracing" => deserialize[ImportVolumeData](jsonValue) + case "updateTdCamera" => deserialize[UpdateTdCamera](jsonValue) + case "createSegment" => deserialize[CreateSegmentVolumeAction](jsonValue) + case "updateSegment" => deserialize[UpdateSegmentVolumeAction](jsonValue) + case "updateSegmentGroups" => deserialize[UpdateSegmentGroupsVolumeAction](jsonValue) + case "deleteSegment" => deserialize[DeleteSegmentVolumeAction](jsonValue) + case "deleteSegmentData" => deserialize[DeleteSegmentDataVolumeAction](jsonValue) + case "updateMappingName" => deserialize[UpdateMappingNameVolumeAction](jsonValue) + case unknownAction: String => JsError(s"Invalid update action s'$unknownAction'") + } // TODO revertToVersion } private def deserialize[T](json: JsValue, shouldTransformPositions: Boolean = false)( diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 8280f681dd8..dcfd07b026d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -37,7 +37,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ MergedVolumeStats, ResolutionRestrictions, TSFullMeshService, - UpdateMappingNameAction, + UpdateMappingNameVolumeAction, VolumeDataZipFormat, VolumeSegmentIndexService, VolumeSegmentStatisticsService, @@ -368,7 +368,7 @@ class VolumeTracingController @Inject()( _ <- bool2Fox(tracingService.volumeBucketsAreEmpty(tracingId)) ?~> "annotation.volumeBucketsNotEmpty" (editableMappingId, editableMappingInfo) <- editableMappingService.create( baseMappingName = tracingMappingName) - volumeUpdate = UpdateMappingNameAction(Some(editableMappingId), + volumeUpdate = UpdateMappingNameVolumeAction(Some(editableMappingId), isEditable = Some(true), isLocked = Some(true), actionTimestamp = Some(System.currentTimeMillis())) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/UpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/UpdateActions.scala index aebd371ae76..878b9cef553 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/UpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/UpdateActions.scala @@ -26,11 +26,6 @@ trait UpdateAction[T <: GeneratedMessage] { def isViewOnlyChange: Boolean = false } -object UpdateAction { - type SkeletonUpdateAction = UpdateAction[SkeletonTracing] - type VolumeUpdateAction = UpdateAction[VolumeTracing] -} - case class UpdateActionGroup[T <: GeneratedMessage]( version: Long, timestamp: Long, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala index 2cb0f710558..c540f85d13b 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala @@ -3,11 +3,10 @@ package com.scalableminds.webknossos.tracingstore.tracings.volume import java.util.Base64 import com.scalableminds.util.geometry.{Vec3Double, Vec3Int} import com.scalableminds.webknossos.datastore.VolumeTracing.{Segment, SegmentGroup, VolumeTracing} -import com.scalableminds.webknossos.datastore.geometry import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate -import com.scalableminds.webknossos.tracingstore.tracings.UpdateAction.VolumeUpdateAction -import com.scalableminds.webknossos.tracingstore.tracings.{NamedBoundingBox, UpdateAction} +import com.scalableminds.webknossos.tracingstore.annotation.GenericUpdateAction +import com.scalableminds.webknossos.tracingstore.tracings.NamedBoundingBox import play.api.libs.json._ trait VolumeUpdateActionHelper { @@ -23,6 +22,8 @@ trait VolumeUpdateActionHelper { } +trait VolumeUpdateAction extends GenericUpdateAction + trait ApplyableVolumeAction extends VolumeUpdateAction case class UpdateBucketVolumeAction(position: Vec3Int, @@ -39,8 +40,9 @@ case class UpdateBucketVolumeAction(position: Vec3Int, override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) - override def transformToCompact: CompactVolumeUpdateAction = + def transformToCompact: CompactVolumeUpdateAction = CompactVolumeUpdateAction("updateBucket", actionTimestamp, actionAuthorId, Json.obj()) } @@ -62,9 +64,7 @@ case class UpdateTracingVolumeAction( override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - - override def transformToCompact: CompactVolumeUpdateAction = - CompactVolumeUpdateAction("updateTracing", actionTimestamp, actionAuthorId, Json.obj()) + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) override def isViewOnlyChange: Boolean = true } @@ -82,11 +82,7 @@ case class RevertToVersionVolumeAction(sourceVersion: Long, override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - override def transformToCompact: CompactVolumeUpdateAction = - CompactVolumeUpdateAction("revertToVersion", - actionTimestamp, - actionAuthorId, - Json.obj("sourceVersion" -> sourceVersion)) + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) } object RevertToVersionVolumeAction { @@ -102,12 +98,10 @@ case class UpdateUserBoundingBoxes(boundingBoxes: List[NamedBoundingBox], this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) - override def transformToCompact: CompactVolumeUpdateAction = - CompactVolumeUpdateAction("updateUserBoundingBoxes", actionTimestamp, actionAuthorId, Json.obj()) - - override def applyOn(tracing: VolumeTracing): VolumeTracing = - tracing.withUserBoundingBoxes(boundingBoxes.map(_.toProto)) + /*override def applyOn(tracing: VolumeTracing): VolumeTracing = + tracing.withUserBoundingBoxes(boundingBoxes.map(_.toProto))*/ } object UpdateUserBoundingBoxes { @@ -123,15 +117,11 @@ case class UpdateUserBoundingBoxVisibility(boundingBoxId: Option[Int], override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) - override def transformToCompact: CompactVolumeUpdateAction = - CompactVolumeUpdateAction("updateUserBoundingBoxVisibility", - actionTimestamp, - actionAuthorId, - Json.obj("boundingBoxId" -> boundingBoxId, "newVisibility" -> isVisible)) override def isViewOnlyChange: Boolean = true - override def applyOn(tracing: VolumeTracing): VolumeTracing = { + /*override def applyOn(tracing: VolumeTracing): VolumeTracing = { def updateUserBoundingBoxes(): Seq[geometry.NamedBoundingBoxProto] = tracing.userBoundingBoxes.map { boundingBox => @@ -142,7 +132,7 @@ case class UpdateUserBoundingBoxVisibility(boundingBoxId: Option[Int], } tracing.withUserBoundingBoxes(updateUserBoundingBoxes()) - } + }*/ } object UpdateUserBoundingBoxVisibility { @@ -156,12 +146,10 @@ case class RemoveFallbackLayer(actionTimestamp: Option[Long] = None, override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) - override def transformToCompact: CompactVolumeUpdateAction = - CompactVolumeUpdateAction("removeFallbackLayer", actionTimestamp, actionAuthorId, Json.obj()) - - override def applyOn(tracing: VolumeTracing): VolumeTracing = - tracing.clearFallbackLayer + /*override def applyOn(tracing: VolumeTracing): VolumeTracing = + tracing.clearFallbackLayer*/ } object RemoveFallbackLayer { @@ -176,15 +164,10 @@ case class ImportVolumeData(largestSegmentId: Option[Long], override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) - override def transformToCompact: CompactVolumeUpdateAction = - CompactVolumeUpdateAction("importVolumeTracing", - actionTimestamp, - actionAuthorId, - Json.obj("largestSegmentId" -> largestSegmentId)) - - override def applyOn(tracing: VolumeTracing): VolumeTracing = - tracing.copy(largestSegmentId = largestSegmentId) + /*override def applyOn(tracing: VolumeTracing): VolumeTracing = + tracing.copy(largestSegmentId = largestSegmentId)*/ } object ImportVolumeData { @@ -196,15 +179,13 @@ case class AddSegmentIndex(actionTimestamp: Option[Long] = None, info: Option[String] = None) extends ApplyableVolumeAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) - override def transformToCompact: CompactVolumeUpdateAction = - CompactVolumeUpdateAction("addSegmentIndex", actionTimestamp, actionAuthorId, Json.obj()) + /*override def applyOn(tracing: VolumeTracing): VolumeTracing = + tracing.copy(hasSegmentIndex = Some(true))*/ - override def applyOn(tracing: VolumeTracing): VolumeTracing = - tracing.copy(hasSegmentIndex = Some(true)) } object AddSegmentIndex { @@ -220,9 +201,7 @@ case class UpdateTdCamera(actionTimestamp: Option[Long] = None, this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - - override def transformToCompact: CompactVolumeUpdateAction = - CompactVolumeUpdateAction("updateTdCamera", actionTimestamp, actionAuthorId, Json.obj()) + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) override def isViewOnlyChange: Boolean = true } @@ -239,7 +218,8 @@ case class CreateSegmentVolumeAction(id: Long, creationTime: Option[Long], actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, - additionalCoordinates: Option[Seq[AdditionalCoordinate]] = None) + additionalCoordinates: Option[Seq[AdditionalCoordinate]] = None, + info: Option[String] = None) extends ApplyableVolumeAction with ProtoGeometryImplicits { @@ -247,11 +227,9 @@ case class CreateSegmentVolumeAction(id: Long, this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) - override def transformToCompact: UpdateAction[VolumeTracing] = - CompactVolumeUpdateAction("createSegment", actionTimestamp, actionAuthorId, Json.obj("id" -> id)) - - override def applyOn(tracing: VolumeTracing): VolumeTracing = { + /*override def applyOn(tracing: VolumeTracing): VolumeTracing = { val newSegment = Segment(id, anchorPosition.map(vec3IntToProto), @@ -261,7 +239,7 @@ case class CreateSegmentVolumeAction(id: Long, groupId, AdditionalCoordinate.toProto(additionalCoordinates)) tracing.addSegments(newSegment) - } + }*/ } object CreateSegmentVolumeAction { @@ -276,7 +254,8 @@ case class UpdateSegmentVolumeAction(id: Long, groupId: Option[Int], actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, - additionalCoordinates: Option[Seq[AdditionalCoordinate]] = None) + additionalCoordinates: Option[Seq[AdditionalCoordinate]] = None, + info: Option[String] = None) extends ApplyableVolumeAction with ProtoGeometryImplicits with VolumeUpdateActionHelper { @@ -285,11 +264,9 @@ case class UpdateSegmentVolumeAction(id: Long, this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) - override def transformToCompact: UpdateAction[VolumeTracing] = - CompactVolumeUpdateAction("updateSegment", actionTimestamp, actionAuthorId, Json.obj("id" -> id)) - - override def applyOn(tracing: VolumeTracing): VolumeTracing = { + /*override def applyOn(tracing: VolumeTracing): VolumeTracing = { def segmentTransform(segment: Segment): Segment = segment.copy( anchorPosition = anchorPosition.map(vec3IntToProto), @@ -300,7 +277,7 @@ case class UpdateSegmentVolumeAction(id: Long, anchorPositionAdditionalCoordinates = AdditionalCoordinate.toProto(additionalCoordinates) ) tracing.withSegments(mapSegments(tracing, id, segmentTransform)) - } + }*/ } object UpdateSegmentVolumeAction { @@ -309,19 +286,18 @@ object UpdateSegmentVolumeAction { case class DeleteSegmentVolumeAction(id: Long, actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None) - extends ApplyableVolumeAction { + actionAuthorId: Option[String] = None, + info: Option[String] = None) + extends VolumeUpdateAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) - override def transformToCompact: UpdateAction[VolumeTracing] = - CompactVolumeUpdateAction("deleteSegment", actionTimestamp, actionAuthorId, Json.obj("id" -> id)) - - override def applyOn(tracing: VolumeTracing): VolumeTracing = - tracing.withSegments(tracing.segments.filter(_.segmentId != id)) + /*override def applyOn(tracing: VolumeTracing): VolumeTracing = + tracing.withSegments(tracing.segments.filter(_.segmentId != id))*/ } @@ -331,53 +307,54 @@ object DeleteSegmentVolumeAction { case class DeleteSegmentDataVolumeAction(id: Long, actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None) + actionAuthorId: Option[String] = None, + info: Option[String] = None) extends VolumeUpdateAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - - override def transformToCompact: CompactVolumeUpdateAction = - CompactVolumeUpdateAction("deleteSegmentData", actionTimestamp, actionAuthorId, Json.obj()) + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) } object DeleteSegmentDataVolumeAction { implicit val jsonFormat: OFormat[DeleteSegmentDataVolumeAction] = Json.format[DeleteSegmentDataVolumeAction] } -case class UpdateMappingNameAction(mappingName: Option[String], - isEditable: Option[Boolean], - isLocked: Option[Boolean], - actionTimestamp: Option[Long], - actionAuthorId: Option[String] = None) +case class UpdateMappingNameVolumeAction(mappingName: Option[String], + isEditable: Option[Boolean], + isLocked: Option[Boolean], + actionTimestamp: Option[Long], + actionAuthorId: Option[String] = None, + info: Option[String] = None) extends ApplyableVolumeAction { - override def addTimestamp(timestamp: Long): VolumeUpdateAction = - this.copy(actionTimestamp = Some(timestamp)) - - override def transformToCompact: UpdateAction[VolumeTracing] = - CompactVolumeUpdateAction("updateMappingName", - actionTimestamp, - actionAuthorId, - Json.obj("mappingName" -> mappingName)) + override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) + override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = + this.copy(actionAuthorId = authorId) + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) - override def applyOn(tracing: VolumeTracing): VolumeTracing = + /* override def applyOn(tracing: VolumeTracing): VolumeTracing = if (tracing.mappingIsLocked.getOrElse(false)) tracing // cannot change mapping name if it is locked else tracing.copy(mappingName = mappingName, mappingIsEditable = Some(isEditable.getOrElse(false)), - mappingIsLocked = Some(isLocked.getOrElse(false))) + mappingIsLocked = Some(isLocked.getOrElse(false)))*/ } -object UpdateMappingNameAction { - implicit val jsonFormat: OFormat[UpdateMappingNameAction] = Json.format[UpdateMappingNameAction] +object UpdateMappingNameVolumeAction { + implicit val jsonFormat: OFormat[UpdateMappingNameVolumeAction] = Json.format[UpdateMappingNameVolumeAction] } case class CompactVolumeUpdateAction(name: String, actionTimestamp: Option[Long], actionAuthorId: Option[String] = None, - value: JsObject) - extends VolumeUpdateAction + value: JsObject, + info: Option[String] = None) + extends VolumeUpdateAction { + override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) + override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = + this.copy(actionAuthorId = authorId) + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) +} object CompactVolumeUpdateAction { implicit object compactVolumeUpdateActionFormat extends Format[CompactVolumeUpdateAction] { @@ -386,8 +363,11 @@ object CompactVolumeUpdateAction { name <- (json \ "name").validate[String] actionTimestamp <- (json \ "value" \ "actionTimestamp").validateOpt[Long] actionAuthorId <- (json \ "value" \ "actionAuthorId").validateOpt[String] - value <- (json \ "value").validate[JsObject].map(_ - "actionTimestamp") - } yield CompactVolumeUpdateAction(name, actionTimestamp, actionAuthorId, value) + info <- (json \ "value" \ "info").validateOpt[String] + value <- (json \ "value") + .validate[JsObject] + .map(_ - "actionTimestamp") // TODO also separate out info + actionAuthorId + } yield CompactVolumeUpdateAction(name, actionTimestamp, actionAuthorId, value, info) override def writes(o: CompactVolumeUpdateAction): JsValue = Json.obj("name" -> o.name, "value" -> (Json.obj("actionTimestamp" -> o.actionTimestamp) ++ o.value)) @@ -398,16 +378,15 @@ case class UpdateSegmentGroupsVolumeAction(segmentGroups: List[UpdateActionSegme actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends ApplyableVolumeAction + extends VolumeUpdateAction with VolumeUpdateActionHelper { - override def applyOn(tracing: VolumeTracing): VolumeTracing = - tracing.withSegmentGroups(segmentGroups.map(convertSegmentGroup)) + /*override def applyOn(tracing: VolumeTracing): VolumeTracing = + tracing.withSegmentGroups(segmentGroups.map(convertSegmentGroup))*/ - override def addTimestamp(timestamp: Long): UpdateAction[VolumeTracing] = - this.copy(actionTimestamp = Some(timestamp)) - override def addAuthorId(authorId: Option[String]): UpdateAction[VolumeTracing] = + override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) + override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): UpdateAction[VolumeTracing] = this.copy(info = info) + override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) } object UpdateSegmentGroupsVolumeAction { @@ -432,7 +411,7 @@ object VolumeUpdateAction { case "updateSegmentGroups" => (json \ "value").validate[UpdateSegmentGroupsVolumeAction] case "deleteSegment" => (json \ "value").validate[DeleteSegmentVolumeAction] case "deleteSegmentData" => (json \ "value").validate[DeleteSegmentDataVolumeAction] - case "updateMappingName" => (json \ "value").validate[UpdateMappingNameAction] + case "updateMappingName" => (json \ "value").validate[UpdateMappingNameVolumeAction] case unknownAction: String => JsError(s"Invalid update action s'$unknownAction'") } From 398bc63030e8839f19e22173cab8ffac75a84b3e Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 16 Jul 2024 13:49:49 +0200 Subject: [PATCH 020/150] cleanup --- .../AnnotationTransactionService.scala | 24 ++-- .../annotation/DSAnnotationService.scala | 13 +- .../annotation/UpdateActions.scala | 39 +++--- .../controllers/DSAnnotationController.scala | 6 +- .../tracingstore/tracings/UpdateActions.scala | 86 ------------- .../updating/SkeletonUpdateActions.scala | 118 +++++++++--------- .../tracings/volume/VolumeUpdateActions.scala | 36 +++--- 7 files changed, 116 insertions(+), 206 deletions(-) delete mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/UpdateActions.scala diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala index 319cd00feab..a29f90e1f0a 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala @@ -34,7 +34,7 @@ class AnnotationTransactionService @Inject()( transactionId: String, transactionGroupIndex: Int, version: Long, - updateGroup: GenericUpdateActionGroup, + updateGroup: UpdateActionGroup, expiry: FiniteDuration)(implicit ec: ExecutionContext): Fox[Unit] = for { _ <- Fox.runIf(transactionGroupIndex > 0)( @@ -52,7 +52,7 @@ class AnnotationTransactionService @Inject()( private def handleUpdateGroupForTransaction(annotationId: String, previousVersionFox: Fox[Long], - updateGroup: GenericUpdateActionGroup, + updateGroup: UpdateActionGroup, userToken: Option[String])(implicit ec: ExecutionContext): Fox[Long] = for { previousCommittedVersion: Long <- previousVersionFox @@ -81,7 +81,7 @@ class AnnotationTransactionService @Inject()( // For an update group (that is the last of a transaction), fetch all previous uncommitted for the same transaction // and commit them all. - private def commitWithPending(annotationId: String, updateGroup: GenericUpdateActionGroup, userToken: Option[String])( + private def commitWithPending(annotationId: String, updateGroup: UpdateActionGroup, userToken: Option[String])( implicit ec: ExecutionContext): Fox[Long] = for { previousActionGroupsToCommit <- getAllUncommittedFor(annotationId, updateGroup.transactionId) @@ -96,11 +96,11 @@ class AnnotationTransactionService @Inject()( private def removeAllUncommittedFor(tracingId: String, transactionId: String): Fox[Unit] = uncommittedUpdatesStore.removeAllConditional(patternFor(tracingId, transactionId)) - private def getAllUncommittedFor(annotationId: String, transactionId: String): Fox[List[GenericUpdateActionGroup]] = + private def getAllUncommittedFor(annotationId: String, transactionId: String): Fox[List[UpdateActionGroup]] = for { raw: Seq[String] <- uncommittedUpdatesStore.findAllConditional(patternFor(annotationId, transactionId)) - parsed: Seq[GenericUpdateActionGroup] = raw.flatMap(itemAsString => - JsonHelper.jsResultToOpt(Json.parse(itemAsString).validate[GenericUpdateActionGroup])) + parsed: Seq[UpdateActionGroup] = raw.flatMap(itemAsString => + JsonHelper.jsResultToOpt(Json.parse(itemAsString).validate[UpdateActionGroup])) } yield parsed.toList.sortBy(_.transactionGroupIndex) private def saveToHandledGroupIdStore(annotationId: String, @@ -118,12 +118,12 @@ class AnnotationTransactionService @Inject()( handledGroupIdStore.contains(handledGroupKey(annotationId, transactionId, version, transactionGroupIndex)) private def concatenateUpdateGroupsOfTransaction( - previousActionGroups: List[GenericUpdateActionGroup], - lastActionGroup: GenericUpdateActionGroup): GenericUpdateActionGroup = + previousActionGroups: List[UpdateActionGroup], + lastActionGroup: UpdateActionGroup): UpdateActionGroup = if (previousActionGroups.isEmpty) lastActionGroup else { val allActionGroups = previousActionGroups :+ lastActionGroup - GenericUpdateActionGroup( + UpdateActionGroup( version = lastActionGroup.version, timestamp = lastActionGroup.timestamp, authorId = lastActionGroup.authorId, @@ -136,7 +136,7 @@ class AnnotationTransactionService @Inject()( ) } - def handleUpdateGroups(annotationId: String, updateGroups: List[GenericUpdateActionGroup], userToken: Option[String])( + def handleUpdateGroups(annotationId: String, updateGroups: List[UpdateActionGroup], userToken: Option[String])( implicit ec: ExecutionContext): Fox[Long] = if (updateGroups.forall(_.transactionGroupCount == 1)) { commitUpdates(annotationId, updateGroups, userToken) @@ -149,7 +149,7 @@ class AnnotationTransactionService @Inject()( // Perform version check and commit the passed updates private def commitUpdates(annotationId: String, - updateGroups: List[GenericUpdateActionGroup], + updateGroups: List[UpdateActionGroup], userToken: Option[String])(implicit ec: ExecutionContext): Fox[Long] = for { _ <- annotationService.reportUpdates(annotationId, updateGroups, userToken) @@ -173,7 +173,7 @@ class AnnotationTransactionService @Inject()( * ignore it silently. This is in case the frontend sends a retry if it believes a save to be unsuccessful * despite the backend receiving it just fine. */ - private def failUnlessAlreadyHandled(updateGroup: GenericUpdateActionGroup, tracingId: String, previousVersion: Long)( + private def failUnlessAlreadyHandled(updateGroup: UpdateActionGroup, tracingId: String, previousVersion: Long)( implicit ec: ExecutionContext): Fox[Long] = { val errorMessage = s"Incorrect version. Expected: ${previousVersion + 1}; Got: ${updateGroup.version}" for { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala index 4813926c08e..e26c5c123d8 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala @@ -10,8 +10,7 @@ import com.scalableminds.webknossos.datastore.Annotation.{ UpdateLayerMetadataAnnotationUpdateAction, UpdateMetadataAnnotationUpdateAction } -import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing -import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore, UpdateActionGroup} +import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingUpdatesReport} import scalapb.GeneratedMessage @@ -23,9 +22,7 @@ class DSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl extends KeyValueStoreImplicits { def storeUpdate(updateAction: GeneratedMessage)(implicit ec: ExecutionContext): Fox[Unit] = Fox.successful(()) - def reportUpdates(annotationId: String, - updateGroups: List[GenericUpdateActionGroup], - userToken: Option[String]): Fox[Unit] = + def reportUpdates(annotationId: String, updateGroups: List[UpdateActionGroup], userToken: Option[String]): Fox[Unit] = for { _ <- remoteWebknossosClient.reportTracingUpdates( TracingUpdatesReport( @@ -41,16 +38,16 @@ class DSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl def currentVersion(annotationId: String): Fox[Long] = ??? def handleUpdateGroup(annotationId: String, - updateActionGroup: GenericUpdateActionGroup, + updateActionGroup: UpdateActionGroup, previousVersion: Long, userToken: Option[String]): Fox[Unit] = - // TODO apply volume updates directly? transform to compact? + // TODO apply some updates directly? transform to compact? tracingDataStore.annotationUpdates.put( annotationId, updateActionGroup.version, updateActionGroup.actions .map(_.addTimestamp(updateActionGroup.timestamp).addAuthorId(updateActionGroup.authorId)) match { //to the first action in the group, attach the group's info - case Nil => List[GenericUpdateAction]() + case Nil => List[UpdateAction]() case first :: rest => first.addInfo(updateActionGroup.info) :: rest } ) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala index 212fd9c4c84..169f6e9d6f1 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala @@ -27,7 +27,6 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ DeleteSegmentVolumeAction, ImportVolumeData, RemoveFallbackLayer, - RevertToVersionVolumeAction, UpdateBucketVolumeAction, UpdateMappingNameVolumeAction, UpdateSegmentGroupsVolumeAction, @@ -39,22 +38,22 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ } import play.api.libs.json.{Format, JsError, JsObject, JsPath, JsResult, JsValue, Json, OFormat, Reads} -trait GenericUpdateAction { +trait UpdateAction { def actionTimestamp: Option[Long] - def addTimestamp(timestamp: Long): GenericUpdateAction + def addTimestamp(timestamp: Long): UpdateAction - def addInfo(info: Option[String]): GenericUpdateAction + def addInfo(info: Option[String]): UpdateAction - def addAuthorId(authorId: Option[String]): GenericUpdateAction + def addAuthorId(authorId: Option[String]): UpdateAction def isViewOnlyChange: Boolean = false } -object GenericUpdateAction { +object UpdateAction { - implicit object genericUpdateActionFormat extends Format[GenericUpdateAction] { - override def reads(json: JsValue): JsResult[GenericUpdateAction] = { + implicit object updateActionFormat extends Format[UpdateAction] { + override def reads(json: JsValue): JsResult[UpdateAction] = { val jsonValue = (json \ "value").as[JsObject] (json \ "name").as[String] match { case "createTree" => deserialize[CreateTreeSkeletonAction](jsonValue) @@ -102,7 +101,7 @@ object GenericUpdateAction { private val positionTransform = (JsPath \ "position").json.update(JsPath.read[List[Float]].map(position => Json.toJson(position.map(_.toInt)))) - override def writes(a: GenericUpdateAction): JsObject = a match { + override def writes(a: UpdateAction): JsObject = a match { case s: CreateTreeSkeletonAction => Json.obj("name" -> "createTree", "value" -> Json.toJson(s)(CreateTreeSkeletonAction.jsonFormat)) case s: DeleteTreeSkeletonAction => @@ -150,21 +149,21 @@ object GenericUpdateAction { } } -case class GenericUpdateActionGroup(version: Long, - timestamp: Long, - authorId: Option[String], - actions: List[GenericUpdateAction], - stats: Option[JsObject], - info: Option[String], - transactionId: String, - transactionGroupCount: Int, - transactionGroupIndex: Int) { +case class UpdateActionGroup(version: Long, + timestamp: Long, + authorId: Option[String], + actions: List[UpdateAction], + stats: Option[JsObject], + info: Option[String], + transactionId: String, + transactionGroupCount: Int, + transactionGroupIndex: Int) { def significantChangesCount: Int = 1 // TODO def viewChangesCount: Int = 1 // TODO } -object GenericUpdateActionGroup { - implicit val jsonFormat: OFormat[GenericUpdateActionGroup] = Json.format[GenericUpdateActionGroup] +object UpdateActionGroup { + implicit val jsonFormat: OFormat[UpdateActionGroup] = Json.format[UpdateActionGroup] } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala index eff6f83c303..d03ea14c69e 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala @@ -6,7 +6,7 @@ import com.scalableminds.webknossos.datastore.controllers.Controller import com.scalableminds.webknossos.datastore.services.UserAccessRequest import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} import com.scalableminds.webknossos.tracingstore.TracingStoreAccessTokenService -import com.scalableminds.webknossos.tracingstore.annotation.{AnnotationTransactionService, GenericUpdateActionGroup} +import com.scalableminds.webknossos.tracingstore.annotation.{AnnotationTransactionService, UpdateActionGroup} import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService import play.api.mvc.{Action, AnyContent, PlayBodyParsers} @@ -31,8 +31,8 @@ class DSAnnotationController @Inject()( } } - def update(annotationId: String, token: Option[String]): Action[List[GenericUpdateActionGroup]] = - Action.async(validateJson[List[GenericUpdateActionGroup]]) { implicit request => + def update(annotationId: String, token: Option[String]): Action[List[UpdateActionGroup]] = + Action.async(validateJson[List[UpdateActionGroup]]) { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { accessTokenService.validateAccess(UserAccessRequest.writeAnnotation(annotationId), diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/UpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/UpdateActions.scala deleted file mode 100644 index 878b9cef553..00000000000 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/UpdateActions.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.scalableminds.webknossos.tracingstore.tracings - -import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing -import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing -import play.api.libs.json._ -import scalapb.GeneratedMessage - -trait UpdateAction[T <: GeneratedMessage] { - - def actionTimestamp: Option[Long] - - def actionAuthorId: Option[String] - - def applyOn(tracing: T): T = tracing - - def addTimestamp(timestamp: Long): UpdateAction[T] = this - - def addInfo(info: Option[String]): UpdateAction[T] = this - - def addAuthorId(authorId: Option[String]): UpdateAction[T] = this - - def transformToCompact: UpdateAction[T] = this - - // For analytics we wan to know how many changes are view only (e.g. move camera, toggle tree visibility) - // Overridden in subclasses - def isViewOnlyChange: Boolean = false -} - -case class UpdateActionGroup[T <: GeneratedMessage]( - version: Long, - timestamp: Long, - authorId: Option[String], - actions: List[UpdateAction[T]], - stats: Option[JsObject], - info: Option[String], - transactionId: String, - transactionGroupCount: Int, - transactionGroupIndex: Int -) { - def significantChangesCount: Int = actions.count(!_.isViewOnlyChange) - def viewChangesCount: Int = actions.count(_.isViewOnlyChange) -} - -object UpdateActionGroup { - - implicit def updateActionGroupReads[T <: GeneratedMessage]( - implicit fmt: Reads[UpdateAction[T]]): Reads[UpdateActionGroup[T]] = - (json: JsValue) => - for { - version <- json.validate((JsPath \ "version").read[Long]) - timestamp <- json.validate((JsPath \ "timestamp").read[Long]) - authorId <- json.validate((JsPath \ "authorId").readNullable[String]) - actions <- json.validate((JsPath \ "actions").read[List[UpdateAction[T]]]) - stats <- json.validate((JsPath \ "stats").readNullable[JsObject]) - info <- json.validate((JsPath \ "info").readNullable[String]) - transactionId <- json.validate((JsPath \ "transactionId").read[String]) - transactionGroupCount <- json.validate((JsPath \ "transactionGroupCount").read[Int]) - transactionGroupIndex <- json.validate((JsPath \ "transactionGroupIndex").read[Int]) - } yield { - UpdateActionGroup[T](version, - timestamp, - authorId, - actions, - stats, - info, - transactionId, - transactionGroupCount, - transactionGroupIndex) - } - - implicit def updateActionGroupWrites[T <: GeneratedMessage]( - implicit fmt: Writes[UpdateAction[T]]): Writes[UpdateActionGroup[T]] = - (value: UpdateActionGroup[T]) => - Json.obj( - "version" -> value.version, - "timestamp" -> value.timestamp, - "authorId" -> value.authorId, - "actions" -> Json.toJson(value.actions), - "stats" -> value.stats, - "info" -> value.info, - "transactionId" -> value.transactionId, - "transactionGroupCount" -> value.transactionGroupCount, - "transactionGroupIndex" -> value.transactionGroupIndex - ) - -} diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala index f449ddd32ec..e24c7d59f68 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala @@ -4,11 +4,11 @@ import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.util.geometry.{Vec3Double, Vec3Int} import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate -import com.scalableminds.webknossos.tracingstore.annotation.GenericUpdateAction +import com.scalableminds.webknossos.tracingstore.annotation.UpdateAction import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.TreeType.TreeType import play.api.libs.json._ -trait SkeletonUpdateAction extends GenericUpdateAction +trait SkeletonUpdateAction extends UpdateAction case class CreateTreeSkeletonAction(id: Int, color: Option[com.scalableminds.util.image.Color], @@ -43,10 +43,10 @@ case class CreateTreeSkeletonAction(id: Int, tracing.withTrees(newTree +: tracing.trees) }*/ - override def addTimestamp(timestamp: Long): GenericUpdateAction = + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): GenericUpdateAction = + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) } @@ -58,12 +58,12 @@ case class DeleteTreeSkeletonAction(id: Int, /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = tracing.withTrees(tracing.trees.filter(_.treeId != id))*/ - override def addTimestamp(timestamp: Long): GenericUpdateAction = + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): GenericUpdateAction = + override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) } @@ -95,10 +95,10 @@ case class UpdateTreeSkeletonAction(id: Int, tracing.withTrees(mapTrees(tracing, id, treeTransform)) }*/ - override def addTimestamp(timestamp: Long): GenericUpdateAction = + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): GenericUpdateAction = + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) } @@ -124,12 +124,12 @@ case class MergeTreeSkeletonAction(sourceId: Int, tracing.withTrees(mapTrees(tracing, targetId, treeTransform).filter(_.treeId != sourceId)) }*/ - override def addTimestamp(timestamp: Long): GenericUpdateAction = + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): GenericUpdateAction = + override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) } @@ -167,12 +167,12 @@ case class MoveTreeComponentSkeletonAction(nodeIds: List[Int], } */ - override def addTimestamp(timestamp: Long): GenericUpdateAction = + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): GenericUpdateAction = + override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) } @@ -189,12 +189,12 @@ case class CreateEdgeSkeletonAction(source: Int, tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) }*/ - override def addTimestamp(timestamp: Long): GenericUpdateAction = + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): GenericUpdateAction = + override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) } @@ -211,12 +211,12 @@ case class DeleteEdgeSkeletonAction(source: Int, tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) }*/ - override def addTimestamp(timestamp: Long): GenericUpdateAction = + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): GenericUpdateAction = + override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) } @@ -257,12 +257,12 @@ case class CreateNodeSkeletonAction(id: Int, tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) }*/ - override def addTimestamp(timestamp: Long): GenericUpdateAction = + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): GenericUpdateAction = + override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) } @@ -306,12 +306,12 @@ case class UpdateNodeSkeletonAction(id: Int, } */ - override def addTimestamp(timestamp: Long): GenericUpdateAction = + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): GenericUpdateAction = + override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) } @@ -331,12 +331,12 @@ case class DeleteNodeSkeletonAction(nodeId: Int, tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) }*/ - override def addTimestamp(timestamp: Long): GenericUpdateAction = + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): GenericUpdateAction = + override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) } @@ -349,12 +349,12 @@ case class UpdateTreeGroupsSkeletonAction(treeGroups: List[UpdateActionTreeGroup /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = tracing.withTreeGroups(treeGroups.map(convertTreeGroup))*/ - override def addTimestamp(timestamp: Long): GenericUpdateAction = + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): GenericUpdateAction = + override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) } @@ -379,12 +379,12 @@ case class UpdateTracingSkeletonAction(activeNode: Option[Int], editPositionAdditionalCoordinates = AdditionalCoordinate.toProto(editPositionAdditionalCoordinates) )*/ - override def addTimestamp(timestamp: Long): GenericUpdateAction = + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): GenericUpdateAction = + override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) } @@ -396,12 +396,12 @@ case class RevertToVersionSkeletonAction(sourceVersion: Long, /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = throw new Exception("RevertToVersionAction applied on unversioned tracing")*/ - override def addTimestamp(timestamp: Long): GenericUpdateAction = + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): GenericUpdateAction = + override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) } @@ -418,12 +418,12 @@ case class UpdateTreeVisibilitySkeletonAction(treeId: Int, tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) }*/ - override def addTimestamp(timestamp: Long): GenericUpdateAction = + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): GenericUpdateAction = + override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) } @@ -456,12 +456,12 @@ case class UpdateTreeGroupVisibilitySkeletonAction(treeGroupId: Option[Int], } }*/ - override def addTimestamp(timestamp: Long): GenericUpdateAction = + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): GenericUpdateAction = + override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) } @@ -479,12 +479,12 @@ case class UpdateTreeEdgesVisibilitySkeletonAction(treeId: Int, tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) }*/ - override def addTimestamp(timestamp: Long): GenericUpdateAction = + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): GenericUpdateAction = + override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) } @@ -496,12 +496,12 @@ case class UpdateUserBoundingBoxesSkeletonAction(boundingBoxes: List[NamedBoundi /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = tracing.withUserBoundingBoxes(boundingBoxes.map(_.toProto))*/ - override def addTimestamp(timestamp: Long): GenericUpdateAction = + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): GenericUpdateAction = + override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) } @@ -523,12 +523,12 @@ case class UpdateUserBoundingBoxVisibilitySkeletonAction(boundingBoxId: Option[I tracing.withUserBoundingBoxes(updateUserBoundingBoxes()) }*/ - override def addTimestamp(timestamp: Long): GenericUpdateAction = + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): GenericUpdateAction = + override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) } @@ -539,12 +539,12 @@ case class UpdateTdCameraSkeletonAction(actionTimestamp: Option[Long] = None, /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = tracing*/ - override def addTimestamp(timestamp: Long): GenericUpdateAction = + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): GenericUpdateAction = + override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala index c540f85d13b..d8a2e36d743 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala @@ -5,7 +5,7 @@ import com.scalableminds.util.geometry.{Vec3Double, Vec3Int} import com.scalableminds.webknossos.datastore.VolumeTracing.{Segment, SegmentGroup, VolumeTracing} import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate -import com.scalableminds.webknossos.tracingstore.annotation.GenericUpdateAction +import com.scalableminds.webknossos.tracingstore.annotation.UpdateAction import com.scalableminds.webknossos.tracingstore.tracings.NamedBoundingBox import play.api.libs.json._ @@ -22,7 +22,7 @@ trait VolumeUpdateActionHelper { } -trait VolumeUpdateAction extends GenericUpdateAction +trait VolumeUpdateAction extends UpdateAction trait ApplyableVolumeAction extends VolumeUpdateAction @@ -40,7 +40,7 @@ case class UpdateBucketVolumeAction(position: Vec3Int, override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) def transformToCompact: CompactVolumeUpdateAction = CompactVolumeUpdateAction("updateBucket", actionTimestamp, actionAuthorId, Json.obj()) @@ -64,7 +64,7 @@ case class UpdateTracingVolumeAction( override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) override def isViewOnlyChange: Boolean = true } @@ -82,7 +82,7 @@ case class RevertToVersionVolumeAction(sourceVersion: Long, override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) } object RevertToVersionVolumeAction { @@ -98,7 +98,7 @@ case class UpdateUserBoundingBoxes(boundingBoxes: List[NamedBoundingBox], this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) /*override def applyOn(tracing: VolumeTracing): VolumeTracing = tracing.withUserBoundingBoxes(boundingBoxes.map(_.toProto))*/ @@ -117,7 +117,7 @@ case class UpdateUserBoundingBoxVisibility(boundingBoxId: Option[Int], override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) override def isViewOnlyChange: Boolean = true @@ -146,7 +146,7 @@ case class RemoveFallbackLayer(actionTimestamp: Option[Long] = None, override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) /*override def applyOn(tracing: VolumeTracing): VolumeTracing = tracing.clearFallbackLayer*/ @@ -164,7 +164,7 @@ case class ImportVolumeData(largestSegmentId: Option[Long], override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) /*override def applyOn(tracing: VolumeTracing): VolumeTracing = tracing.copy(largestSegmentId = largestSegmentId)*/ @@ -181,7 +181,7 @@ case class AddSegmentIndex(actionTimestamp: Option[Long] = None, override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) /*override def applyOn(tracing: VolumeTracing): VolumeTracing = tracing.copy(hasSegmentIndex = Some(true))*/ @@ -201,7 +201,7 @@ case class UpdateTdCamera(actionTimestamp: Option[Long] = None, this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) override def isViewOnlyChange: Boolean = true } @@ -227,7 +227,7 @@ case class CreateSegmentVolumeAction(id: Long, this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) /*override def applyOn(tracing: VolumeTracing): VolumeTracing = { val newSegment = @@ -264,7 +264,7 @@ case class UpdateSegmentVolumeAction(id: Long, this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) /*override def applyOn(tracing: VolumeTracing): VolumeTracing = { def segmentTransform(segment: Segment): Segment = @@ -294,7 +294,7 @@ case class DeleteSegmentVolumeAction(id: Long, this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) /*override def applyOn(tracing: VolumeTracing): VolumeTracing = tracing.withSegments(tracing.segments.filter(_.segmentId != id))*/ @@ -313,7 +313,7 @@ case class DeleteSegmentDataVolumeAction(id: Long, override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) } object DeleteSegmentDataVolumeAction { @@ -330,7 +330,7 @@ case class UpdateMappingNameVolumeAction(mappingName: Option[String], override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) /* override def applyOn(tracing: VolumeTracing): VolumeTracing = if (tracing.mappingIsLocked.getOrElse(false)) tracing // cannot change mapping name if it is locked @@ -353,7 +353,7 @@ case class CompactVolumeUpdateAction(name: String, override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) } object CompactVolumeUpdateAction { @@ -386,7 +386,7 @@ case class UpdateSegmentGroupsVolumeAction(segmentGroups: List[UpdateActionSegme override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): GenericUpdateAction = this.copy(info = info) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) } object UpdateSegmentGroupsVolumeAction { From 08b8bccf22f74069ef3a49f848580877075e360e Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 17 Jul 2024 11:24:35 +0200 Subject: [PATCH 021/150] volume action serialization, wip annotation actions --- .../annotation/AnnotationUpdateActions.scala | 75 ++++++ .../annotation/DSAnnotationService.scala | 27 ++- .../annotation/UpdateActions.scala | 72 +++++- .../EditableMappingUpdateActions.scala | 65 ++--- .../volume/VolumeTracingService.scala | 6 +- .../tracings/volume/VolumeUpdateActions.scala | 222 ++++++------------ 6 files changed, 250 insertions(+), 217 deletions(-) create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala new file mode 100644 index 00000000000..725634f66e8 --- /dev/null +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala @@ -0,0 +1,75 @@ +package com.scalableminds.webknossos.tracingstore.annotation + +import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType.AnnotationLayerType +import play.api.libs.json.{Json, OFormat} + +trait AnnotationUpdateAction extends UpdateAction + +case class AddLayerAnnotationUpdateAction(layerName: String, + tracingId: String, + typ: AnnotationLayerType, + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) + extends AnnotationUpdateAction { + override def addTimestamp(timestamp: Long): UpdateAction = + this.copy(actionTimestamp = Some(timestamp)) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def addAuthorId(authorId: Option[String]): UpdateAction = + this.copy(actionAuthorId = authorId) +} + +case class DeleteLayerAnnotationUpdateAction(tracingId: String, + layerName: String, // Just stored for nicer-looking history + typ: AnnotationLayerType, // Just stored for nicer-looking history + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) + extends AnnotationUpdateAction { + override def addTimestamp(timestamp: Long): UpdateAction = + this.copy(actionTimestamp = Some(timestamp)) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def addAuthorId(authorId: Option[String]): UpdateAction = + this.copy(actionAuthorId = authorId) +} + +case class UpdateLayerMetadataAnnotationUpdateAction(tracingId: String, + layerName: String, // Just stored for nicer-looking history + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) + extends AnnotationUpdateAction { + override def addTimestamp(timestamp: Long): UpdateAction = + this.copy(actionTimestamp = Some(timestamp)) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def addAuthorId(authorId: Option[String]): UpdateAction = + this.copy(actionAuthorId = authorId) +} + +case class UpdateMetadataAnnotationUpdateAction(name: Option[String], + description: Option[String], + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) + extends AnnotationUpdateAction { + override def addTimestamp(timestamp: Long): UpdateAction = + this.copy(actionTimestamp = Some(timestamp)) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def addAuthorId(authorId: Option[String]): UpdateAction = + this.copy(actionAuthorId = authorId) +} + +object AddLayerAnnotationUpdateAction { + implicit val jsonFormat: OFormat[AddLayerAnnotationUpdateAction] = Json.format[AddLayerAnnotationUpdateAction] +} +object DeleteLayerAnnotationUpdateAction { + implicit val jsonFormat: OFormat[DeleteLayerAnnotationUpdateAction] = Json.format[DeleteLayerAnnotationUpdateAction] +} +object UpdateLayerMetadataAnnotationUpdateAction { + implicit val jsonFormat: OFormat[UpdateLayerMetadataAnnotationUpdateAction] = + Json.format[UpdateLayerMetadataAnnotationUpdateAction] +} +object UpdateMetadataAnnotationUpdateAction { + implicit val jsonFormat: OFormat[UpdateMetadataAnnotationUpdateAction] = + Json.format[UpdateMetadataAnnotationUpdateAction] +} diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala index e26c5c123d8..c0c634e5f74 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala @@ -10,6 +10,7 @@ import com.scalableminds.webknossos.datastore.Annotation.{ UpdateLayerMetadataAnnotationUpdateAction, UpdateMetadataAnnotationUpdateAction } +import com.scalableminds.webknossos.tracingstore.tracings.volume.UpdateBucketVolumeAction import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingUpdatesReport} import scalapb.GeneratedMessage @@ -20,7 +21,6 @@ import scala.concurrent.ExecutionContext class DSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosClient, tracingDataStore: TracingDataStore) extends KeyValueStoreImplicits { - def storeUpdate(updateAction: GeneratedMessage)(implicit ec: ExecutionContext): Fox[Unit] = Fox.successful(()) def reportUpdates(annotationId: String, updateGroups: List[UpdateActionGroup], userToken: Option[String]): Fox[Unit] = for { @@ -42,15 +42,22 @@ class DSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl previousVersion: Long, userToken: Option[String]): Fox[Unit] = // TODO apply some updates directly? transform to compact? - tracingDataStore.annotationUpdates.put( - annotationId, - updateActionGroup.version, - updateActionGroup.actions - .map(_.addTimestamp(updateActionGroup.timestamp).addAuthorId(updateActionGroup.authorId)) match { //to the first action in the group, attach the group's info - case Nil => List[UpdateAction]() - case first :: rest => first.addInfo(updateActionGroup.info) :: rest - } - ) + tracingDataStore.annotationUpdates.put(annotationId, + updateActionGroup.version, + preprocessActionsForStorage(updateActionGroup)) + + private def preprocessActionsForStorage(updateActionGroup: UpdateActionGroup): List[UpdateAction] = { + val actionsWithInfo = updateActionGroup.actions.map( + _.addTimestamp(updateActionGroup.timestamp).addAuthorId(updateActionGroup.authorId)) match { + case Nil => List[UpdateAction]() + //to the first action in the group, attach the group's info + case first :: rest => first.addInfo(updateActionGroup.info) :: rest + } + actionsWithInfo.map { + case a: UpdateBucketVolumeAction => a.transformToCompact // TODO or not? + case a => a + } + } def applyUpdate(annotation: AnnotationProto, updateAction: GeneratedMessage)( implicit ec: ExecutionContext): Fox[AnnotationProto] = diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala index 169f6e9d6f1..1858251c968 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala @@ -1,5 +1,9 @@ package com.scalableminds.webknossos.tracingstore.annotation +import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ + MergeAgglomerateUpdateAction, + SplitAgglomerateUpdateAction +} import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{ CreateEdgeSkeletonAction, CreateNodeSkeletonAction, @@ -22,19 +26,20 @@ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{ UpdateUserBoundingBoxesSkeletonAction } import com.scalableminds.webknossos.tracingstore.tracings.volume.{ + CompactVolumeUpdateAction, CreateSegmentVolumeAction, DeleteSegmentDataVolumeAction, DeleteSegmentVolumeAction, - ImportVolumeData, - RemoveFallbackLayer, + ImportVolumeDataVolumeAction, + RemoveFallbackLayerVolumeAction, UpdateBucketVolumeAction, UpdateMappingNameVolumeAction, UpdateSegmentGroupsVolumeAction, UpdateSegmentVolumeAction, - UpdateTdCamera, + UpdateTdCameraVolumeAction, UpdateTracingVolumeAction, - UpdateUserBoundingBoxVisibility, - UpdateUserBoundingBoxes + UpdateUserBoundingBoxVisibilityVolumeAction, + UpdateUserBoundingBoxesVolumeAction } import play.api.libs.json.{Format, JsError, JsObject, JsPath, JsResult, JsValue, Json, OFormat, Reads} @@ -56,6 +61,7 @@ object UpdateAction { override def reads(json: JsValue): JsResult[UpdateAction] = { val jsonValue = (json \ "value").as[JsObject] (json \ "name").as[String] match { + // Skeletons case "createTree" => deserialize[CreateTreeSkeletonAction](jsonValue) case "deleteTree" => deserialize[DeleteTreeSkeletonAction](jsonValue) case "updateTree" => deserialize[UpdateTreeSkeletonAction](jsonValue) @@ -74,21 +80,30 @@ object UpdateAction { case "updateUserBoundingBoxes" => deserialize[UpdateUserBoundingBoxesSkeletonAction](jsonValue) case "updateUserBoundingBoxVisibility" => deserialize[UpdateUserBoundingBoxVisibilitySkeletonAction](jsonValue) + + // Volumes case "updateBucket" => deserialize[UpdateBucketVolumeAction](jsonValue) case "updateVolumeTracing" => deserialize[UpdateTracingVolumeAction](jsonValue) - case "updateUserBoundingBoxes" => deserialize[UpdateUserBoundingBoxes](jsonValue) - case "updateUserBoundingBoxVisibility" => deserialize[UpdateUserBoundingBoxVisibility](jsonValue) - case "removeFallbackLayer" => deserialize[RemoveFallbackLayer](jsonValue) - case "importVolumeTracing" => deserialize[ImportVolumeData](jsonValue) - case "updateTdCamera" => deserialize[UpdateTdCamera](jsonValue) + case "updateUserBoundingBoxes" => deserialize[UpdateUserBoundingBoxesVolumeAction](jsonValue) + case "updateUserBoundingBoxVisibility" => deserialize[UpdateUserBoundingBoxVisibilityVolumeAction](jsonValue) + case "removeFallbackLayer" => deserialize[RemoveFallbackLayerVolumeAction](jsonValue) + case "importVolumeTracing" => deserialize[ImportVolumeDataVolumeAction](jsonValue) + case "updateTdCamera" => deserialize[UpdateTdCameraVolumeAction](jsonValue) case "createSegment" => deserialize[CreateSegmentVolumeAction](jsonValue) case "updateSegment" => deserialize[UpdateSegmentVolumeAction](jsonValue) case "updateSegmentGroups" => deserialize[UpdateSegmentGroupsVolumeAction](jsonValue) case "deleteSegment" => deserialize[DeleteSegmentVolumeAction](jsonValue) case "deleteSegmentData" => deserialize[DeleteSegmentDataVolumeAction](jsonValue) case "updateMappingName" => deserialize[UpdateMappingNameVolumeAction](jsonValue) - case unknownAction: String => JsError(s"Invalid update action s'$unknownAction'") - } // TODO revertToVersion + + // Editable Mappings + case "mergeAgglomerate" => deserialize[MergeAgglomerateUpdateAction](jsonValue) + case "splitAgglomerate" => deserialize[SplitAgglomerateUpdateAction](jsonValue) + + // TODO: Annotation, RevertToVersion + + case unknownAction: String => JsError(s"Invalid update action s'$unknownAction'") + } } private def deserialize[T](json: JsValue, shouldTransformPositions: Boolean = false)( @@ -101,7 +116,7 @@ object UpdateAction { private val positionTransform = (JsPath \ "position").json.update(JsPath.read[List[Float]].map(position => Json.toJson(position.map(_.toInt)))) - override def writes(a: UpdateAction): JsObject = a match { + override def writes(a: UpdateAction): JsValue = a match { case s: CreateTreeSkeletonAction => Json.obj("name" -> "createTree", "value" -> Json.toJson(s)(CreateTreeSkeletonAction.jsonFormat)) case s: DeleteTreeSkeletonAction => @@ -145,6 +160,37 @@ object UpdateAction { "value" -> Json.toJson(s)(UpdateUserBoundingBoxVisibilitySkeletonAction.jsonFormat)) case s: UpdateTdCameraSkeletonAction => Json.obj("name" -> "updateTdCamera", "value" -> Json.toJson(s)(UpdateTdCameraSkeletonAction.jsonFormat)) + + case s: UpdateBucketVolumeAction => + Json.obj("name" -> "updateBucket", "value" -> Json.toJson(s)(UpdateBucketVolumeAction.jsonFormat)) + case s: UpdateTracingVolumeAction => + Json.obj("name" -> "updateTracing", "value" -> Json.toJson(s)(UpdateTracingVolumeAction.jsonFormat)) + case s: UpdateUserBoundingBoxesVolumeAction => + Json.obj("name" -> "updateUserBoundingBoxes", + "value" -> Json.toJson(s)(UpdateUserBoundingBoxesVolumeAction.jsonFormat)) + case s: UpdateUserBoundingBoxVisibilityVolumeAction => + Json.obj("name" -> "updateUserBoundingBoxVisibility", + "value" -> Json.toJson(s)(UpdateUserBoundingBoxVisibilityVolumeAction.jsonFormat)) + case s: RemoveFallbackLayerVolumeAction => + Json.obj("name" -> "removeFallbackLayer", "value" -> Json.toJson(s)(RemoveFallbackLayerVolumeAction.jsonFormat)) + case s: ImportVolumeDataVolumeAction => + Json.obj("name" -> "importVolumeTracing", "value" -> Json.toJson(s)(ImportVolumeDataVolumeAction.jsonFormat)) + case s: UpdateTdCameraVolumeAction => + Json.obj("name" -> "updateTdCamera", "value" -> Json.toJson(s)(UpdateTdCameraVolumeAction.jsonFormat)) + case s: CreateSegmentVolumeAction => + Json.obj("name" -> "createSegment", "value" -> Json.toJson(s)(CreateSegmentVolumeAction.jsonFormat)) + case s: UpdateSegmentVolumeAction => + Json.obj("name" -> "updateSegment", "value" -> Json.toJson(s)(UpdateSegmentVolumeAction.jsonFormat)) + case s: DeleteSegmentVolumeAction => + Json.obj("name" -> "deleteSegment", "value" -> Json.toJson(s)(DeleteSegmentVolumeAction.jsonFormat)) + case s: UpdateSegmentGroupsVolumeAction => + Json.obj("name" -> "updateSegmentGroups", "value" -> Json.toJson(s)(UpdateSegmentGroupsVolumeAction.jsonFormat)) + case s: CompactVolumeUpdateAction => Json.toJson(s)(CompactVolumeUpdateAction.compactVolumeUpdateActionFormat) + + case s: SplitAgglomerateUpdateAction => + Json.obj("name" -> "splitAgglomerate", "value" -> Json.toJson(s)(SplitAgglomerateUpdateAction.jsonFormat)) + case s: MergeAgglomerateUpdateAction => + Json.obj("name" -> "mergeAgglomerate", "value" -> Json.toJson(s)(MergeAgglomerateUpdateAction.jsonFormat)) } } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala index 96f01b912da..7f037238097 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala @@ -1,12 +1,11 @@ package com.scalableminds.webknossos.tracingstore.tracings.editablemapping import com.scalableminds.util.geometry.Vec3Int +import com.scalableminds.webknossos.tracingstore.annotation.UpdateAction import play.api.libs.json.Format.GenericFormat import play.api.libs.json._ -trait EditableMappingUpdateAction { - def addTimestamp(timestamp: Long): EditableMappingUpdateAction -} +trait EditableMappingUpdateAction extends UpdateAction // we switched from positions to segment ids in https://github.com/scalableminds/webknossos/pull/7742. // Both are now optional to support applying old update actions stored in the db. @@ -16,9 +15,16 @@ case class SplitAgglomerateUpdateAction(agglomerateId: Long, segmentId1: Option[Long], segmentId2: Option[Long], mag: Vec3Int, - actionTimestamp: Option[Long] = None) + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) extends EditableMappingUpdateAction { override def addTimestamp(timestamp: Long): EditableMappingUpdateAction = this.copy(actionTimestamp = Some(timestamp)) + + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + + override def addAuthorId(authorId: Option[String]): UpdateAction = + this.copy(actionAuthorId = authorId) } object SplitAgglomerateUpdateAction { @@ -34,53 +40,18 @@ case class MergeAgglomerateUpdateAction(agglomerateId1: Long, segmentId1: Option[Long], segmentId2: Option[Long], mag: Vec3Int, - actionTimestamp: Option[Long] = None) + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) extends EditableMappingUpdateAction { override def addTimestamp(timestamp: Long): EditableMappingUpdateAction = this.copy(actionTimestamp = Some(timestamp)) -} -object MergeAgglomerateUpdateAction { - implicit val jsonFormat: OFormat[MergeAgglomerateUpdateAction] = Json.format[MergeAgglomerateUpdateAction] -} + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) -case class RevertToVersionUpdateAction(sourceVersion: Long, actionTimestamp: Option[Long] = None) - extends EditableMappingUpdateAction { - override def addTimestamp(timestamp: Long): EditableMappingUpdateAction = this.copy(actionTimestamp = Some(timestamp)) + override def addAuthorId(authorId: Option[String]): UpdateAction = + this.copy(actionAuthorId = authorId) } -object RevertToVersionUpdateAction { - implicit val jsonFormat: OFormat[RevertToVersionUpdateAction] = Json.format[RevertToVersionUpdateAction] -} - -object EditableMappingUpdateAction { - - implicit object editableMappingUpdateActionFormat extends Format[EditableMappingUpdateAction] { - override def reads(json: JsValue): JsResult[EditableMappingUpdateAction] = - (json \ "name").validate[String].flatMap { - case "mergeAgglomerate" => (json \ "value").validate[MergeAgglomerateUpdateAction] - case "splitAgglomerate" => (json \ "value").validate[SplitAgglomerateUpdateAction] - case "revertToVersion" => (json \ "value").validate[RevertToVersionUpdateAction] - case unknownAction: String => JsError(s"Invalid update action s'$unknownAction'") - } - - override def writes(o: EditableMappingUpdateAction): JsValue = o match { - case s: SplitAgglomerateUpdateAction => - Json.obj("name" -> "splitAgglomerate", "value" -> Json.toJson(s)(SplitAgglomerateUpdateAction.jsonFormat)) - case s: MergeAgglomerateUpdateAction => - Json.obj("name" -> "mergeAgglomerate", "value" -> Json.toJson(s)(MergeAgglomerateUpdateAction.jsonFormat)) - case s: RevertToVersionUpdateAction => - Json.obj("name" -> "revertToVersion", "value" -> Json.toJson(s)(RevertToVersionUpdateAction.jsonFormat)) - } - } - -} - -case class EditableMappingUpdateActionGroup( - version: Long, - timestamp: Long, - actions: List[EditableMappingUpdateAction] -) - -object EditableMappingUpdateActionGroup { - implicit val jsonFormat: OFormat[EditableMappingUpdateActionGroup] = Json.format[EditableMappingUpdateActionGroup] +object MergeAgglomerateUpdateAction { + implicit val jsonFormat: OFormat[MergeAgglomerateUpdateAction] = Json.format[MergeAgglomerateUpdateAction] } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 74c46726eae..7a19b438837 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -162,7 +162,7 @@ class VolumeTracingService @Inject()( Fox.failure("Cannot delete segment data for annotations without segment index.") } else deleteSegmentData(tracingId, tracing, a, segmentIndexBuffer, updateGroup.version, userToken) ?~> "Failed to delete segment data." - case _: UpdateTdCamera => Fox.successful(tracing) + case _: UpdateTdCameraVolumeAction => Fox.successful(tracing) case a: ApplyableVolumeAction => Fox.successful(a.applyOn(tracing)) case _ => Fox.failure("Unknown action.") } @@ -909,7 +909,7 @@ class VolumeTracingService @Inject()( tracing.version + 1L, System.currentTimeMillis(), None, - List(AddSegmentIndex()), + List(AddSegmentIndexVolumeAction()), None, None, "dummyTransactionId", @@ -996,7 +996,7 @@ class VolumeTracingService @Inject()( tracing.version + 1, System.currentTimeMillis(), None, - List(ImportVolumeData(Some(mergedVolume.largestSegmentId.toPositiveLong))), + List(ImportVolumeDataVolumeAction(Some(mergedVolume.largestSegmentId.toPositiveLong))), None, None, "dummyTransactionId", diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala index d8a2e36d743..360bd55c055 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala @@ -46,10 +46,6 @@ case class UpdateBucketVolumeAction(position: Vec3Int, CompactVolumeUpdateAction("updateBucket", actionTimestamp, actionAuthorId, Json.obj()) } -object UpdateBucketVolumeAction { - implicit val jsonFormat: OFormat[UpdateBucketVolumeAction] = Json.format[UpdateBucketVolumeAction] -} - case class UpdateTracingVolumeAction( activeSegmentId: Long, editPosition: Vec3Int, @@ -69,10 +65,6 @@ case class UpdateTracingVolumeAction( override def isViewOnlyChange: Boolean = true } -object UpdateTracingVolumeAction { - implicit val jsonFormat: OFormat[UpdateTracingVolumeAction] = Json.format[UpdateTracingVolumeAction] -} - case class RevertToVersionVolumeAction(sourceVersion: Long, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, @@ -85,14 +77,10 @@ case class RevertToVersionVolumeAction(sourceVersion: Long, override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) } -object RevertToVersionVolumeAction { - implicit val jsonFormat: OFormat[RevertToVersionVolumeAction] = Json.format[RevertToVersionVolumeAction] -} - -case class UpdateUserBoundingBoxes(boundingBoxes: List[NamedBoundingBox], - actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) +case class UpdateUserBoundingBoxesVolumeAction(boundingBoxes: List[NamedBoundingBox], + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) extends ApplyableVolumeAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -104,15 +92,11 @@ case class UpdateUserBoundingBoxes(boundingBoxes: List[NamedBoundingBox], tracing.withUserBoundingBoxes(boundingBoxes.map(_.toProto))*/ } -object UpdateUserBoundingBoxes { - implicit val jsonFormat: OFormat[UpdateUserBoundingBoxes] = Json.format[UpdateUserBoundingBoxes] -} - -case class UpdateUserBoundingBoxVisibility(boundingBoxId: Option[Int], - isVisible: Boolean, - actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) +case class UpdateUserBoundingBoxVisibilityVolumeAction(boundingBoxId: Option[Int], + isVisible: Boolean, + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) extends ApplyableVolumeAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = @@ -135,13 +119,9 @@ case class UpdateUserBoundingBoxVisibility(boundingBoxId: Option[Int], }*/ } -object UpdateUserBoundingBoxVisibility { - implicit val jsonFormat: OFormat[UpdateUserBoundingBoxVisibility] = Json.format[UpdateUserBoundingBoxVisibility] -} - -case class RemoveFallbackLayer(actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) +case class RemoveFallbackLayerVolumeAction(actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) extends ApplyableVolumeAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = @@ -152,14 +132,10 @@ case class RemoveFallbackLayer(actionTimestamp: Option[Long] = None, tracing.clearFallbackLayer*/ } -object RemoveFallbackLayer { - implicit val jsonFormat: OFormat[RemoveFallbackLayer] = Json.format[RemoveFallbackLayer] -} - -case class ImportVolumeData(largestSegmentId: Option[Long], - actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) +case class ImportVolumeDataVolumeAction(largestSegmentId: Option[Long], + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) extends ApplyableVolumeAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = @@ -170,13 +146,9 @@ case class ImportVolumeData(largestSegmentId: Option[Long], tracing.copy(largestSegmentId = largestSegmentId)*/ } -object ImportVolumeData { - implicit val jsonFormat: OFormat[ImportVolumeData] = Json.format[ImportVolumeData] -} - -case class AddSegmentIndex(actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) +case class AddSegmentIndexVolumeAction(actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) extends ApplyableVolumeAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = @@ -188,13 +160,9 @@ case class AddSegmentIndex(actionTimestamp: Option[Long] = None, } -object AddSegmentIndex { - implicit val jsonFormat: OFormat[AddSegmentIndex] = Json.format[AddSegmentIndex] -} - -case class UpdateTdCamera(actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) +case class UpdateTdCameraVolumeAction(actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) extends VolumeUpdateAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = @@ -206,10 +174,6 @@ case class UpdateTdCamera(actionTimestamp: Option[Long] = None, override def isViewOnlyChange: Boolean = true } -object UpdateTdCamera { - implicit val jsonFormat: OFormat[UpdateTdCamera] = Json.format[UpdateTdCamera] -} - case class CreateSegmentVolumeAction(id: Long, anchorPosition: Option[Vec3Int], name: Option[String], @@ -242,10 +206,6 @@ case class CreateSegmentVolumeAction(id: Long, }*/ } -object CreateSegmentVolumeAction { - implicit val jsonFormat: OFormat[CreateSegmentVolumeAction] = Json.format[CreateSegmentVolumeAction] -} - case class UpdateSegmentVolumeAction(id: Long, anchorPosition: Option[Vec3Int], name: Option[String], @@ -280,10 +240,6 @@ case class UpdateSegmentVolumeAction(id: Long, }*/ } -object UpdateSegmentVolumeAction { - implicit val jsonFormat: OFormat[UpdateSegmentVolumeAction] = Json.format[UpdateSegmentVolumeAction] -} - case class DeleteSegmentVolumeAction(id: Long, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, @@ -301,10 +257,6 @@ case class DeleteSegmentVolumeAction(id: Long, } -object DeleteSegmentVolumeAction { - implicit val jsonFormat: OFormat[DeleteSegmentVolumeAction] = Json.format[DeleteSegmentVolumeAction] -} - case class DeleteSegmentDataVolumeAction(id: Long, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, @@ -316,10 +268,6 @@ case class DeleteSegmentDataVolumeAction(id: Long, override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) } -object DeleteSegmentDataVolumeAction { - implicit val jsonFormat: OFormat[DeleteSegmentDataVolumeAction] = Json.format[DeleteSegmentDataVolumeAction] -} - case class UpdateMappingNameVolumeAction(mappingName: Option[String], isEditable: Option[Boolean], isLocked: Option[Boolean], @@ -340,8 +288,19 @@ case class UpdateMappingNameVolumeAction(mappingName: Option[String], mappingIsLocked = Some(isLocked.getOrElse(false)))*/ } -object UpdateMappingNameVolumeAction { - implicit val jsonFormat: OFormat[UpdateMappingNameVolumeAction] = Json.format[UpdateMappingNameVolumeAction] +case class UpdateSegmentGroupsVolumeAction(segmentGroups: List[UpdateActionSegmentGroup], + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) + extends VolumeUpdateAction + with VolumeUpdateActionHelper { + /*override def applyOn(tracing: VolumeTracing): VolumeTracing = + tracing.withSegmentGroups(segmentGroups.map(convertSegmentGroup))*/ + + override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) + override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = + this.copy(actionAuthorId = authorId) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) } case class CompactVolumeUpdateAction(name: String, @@ -374,75 +333,50 @@ object CompactVolumeUpdateAction { } } -case class UpdateSegmentGroupsVolumeAction(segmentGroups: List[UpdateActionSegmentGroup], - actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) - extends VolumeUpdateAction - with VolumeUpdateActionHelper { - /*override def applyOn(tracing: VolumeTracing): VolumeTracing = - tracing.withSegmentGroups(segmentGroups.map(convertSegmentGroup))*/ - - override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = - this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) +object UpdateBucketVolumeAction { + implicit val jsonFormat: OFormat[UpdateBucketVolumeAction] = Json.format[UpdateBucketVolumeAction] +} +object UpdateTracingVolumeAction { + implicit val jsonFormat: OFormat[UpdateTracingVolumeAction] = Json.format[UpdateTracingVolumeAction] +} +object RevertToVersionVolumeAction { + implicit val jsonFormat: OFormat[RevertToVersionVolumeAction] = Json.format[RevertToVersionVolumeAction] +} +object UpdateUserBoundingBoxesVolumeAction { + implicit val jsonFormat: OFormat[UpdateUserBoundingBoxesVolumeAction] = + Json.format[UpdateUserBoundingBoxesVolumeAction] +} +object UpdateUserBoundingBoxVisibilityVolumeAction { + implicit val jsonFormat: OFormat[UpdateUserBoundingBoxVisibilityVolumeAction] = + Json.format[UpdateUserBoundingBoxVisibilityVolumeAction] +} +object RemoveFallbackLayerVolumeAction { + implicit val jsonFormat: OFormat[RemoveFallbackLayerVolumeAction] = Json.format[RemoveFallbackLayerVolumeAction] +} +object ImportVolumeDataVolumeAction { + implicit val jsonFormat: OFormat[ImportVolumeDataVolumeAction] = Json.format[ImportVolumeDataVolumeAction] +} +object AddSegmentIndexVolumeAction { + implicit val jsonFormat: OFormat[AddSegmentIndexVolumeAction] = Json.format[AddSegmentIndexVolumeAction] +} +object UpdateTdCameraVolumeAction { + implicit val jsonFormat: OFormat[UpdateTdCameraVolumeAction] = Json.format[UpdateTdCameraVolumeAction] +} +object CreateSegmentVolumeAction { + implicit val jsonFormat: OFormat[CreateSegmentVolumeAction] = Json.format[CreateSegmentVolumeAction] +} +object UpdateSegmentVolumeAction { + implicit val jsonFormat: OFormat[UpdateSegmentVolumeAction] = Json.format[UpdateSegmentVolumeAction] +} +object DeleteSegmentVolumeAction { + implicit val jsonFormat: OFormat[DeleteSegmentVolumeAction] = Json.format[DeleteSegmentVolumeAction] +} +object DeleteSegmentDataVolumeAction { + implicit val jsonFormat: OFormat[DeleteSegmentDataVolumeAction] = Json.format[DeleteSegmentDataVolumeAction] +} +object UpdateMappingNameVolumeAction { + implicit val jsonFormat: OFormat[UpdateMappingNameVolumeAction] = Json.format[UpdateMappingNameVolumeAction] } - object UpdateSegmentGroupsVolumeAction { implicit val jsonFormat: OFormat[UpdateSegmentGroupsVolumeAction] = Json.format[UpdateSegmentGroupsVolumeAction] } - -object VolumeUpdateAction { - - implicit object volumeUpdateActionFormat extends Format[VolumeUpdateAction] { - override def reads(json: JsValue): JsResult[VolumeUpdateAction] = - (json \ "name").validate[String].flatMap { - case "updateBucket" => (json \ "value").validate[UpdateBucketVolumeAction] - case "updateTracing" => (json \ "value").validate[UpdateTracingVolumeAction] - case "revertToVersion" => (json \ "value").validate[RevertToVersionVolumeAction] - case "updateUserBoundingBoxes" => (json \ "value").validate[UpdateUserBoundingBoxes] - case "updateUserBoundingBoxVisibility" => (json \ "value").validate[UpdateUserBoundingBoxVisibility] - case "removeFallbackLayer" => (json \ "value").validate[RemoveFallbackLayer] - case "importVolumeTracing" => (json \ "value").validate[ImportVolumeData] - case "updateTdCamera" => (json \ "value").validate[UpdateTdCamera] - case "createSegment" => (json \ "value").validate[CreateSegmentVolumeAction] - case "updateSegment" => (json \ "value").validate[UpdateSegmentVolumeAction] - case "updateSegmentGroups" => (json \ "value").validate[UpdateSegmentGroupsVolumeAction] - case "deleteSegment" => (json \ "value").validate[DeleteSegmentVolumeAction] - case "deleteSegmentData" => (json \ "value").validate[DeleteSegmentDataVolumeAction] - case "updateMappingName" => (json \ "value").validate[UpdateMappingNameVolumeAction] - case unknownAction: String => JsError(s"Invalid update action s'$unknownAction'") - } - - override def writes(o: VolumeUpdateAction): JsValue = o match { - case s: UpdateBucketVolumeAction => - Json.obj("name" -> "updateBucket", "value" -> Json.toJson(s)(UpdateBucketVolumeAction.jsonFormat)) - case s: UpdateTracingVolumeAction => - Json.obj("name" -> "updateTracing", "value" -> Json.toJson(s)(UpdateTracingVolumeAction.jsonFormat)) - case s: RevertToVersionVolumeAction => - Json.obj("name" -> "revertToVersion", "value" -> Json.toJson(s)(RevertToVersionVolumeAction.jsonFormat)) - case s: UpdateUserBoundingBoxes => - Json.obj("name" -> "updateUserBoundingBoxes", "value" -> Json.toJson(s)(UpdateUserBoundingBoxes.jsonFormat)) - case s: UpdateUserBoundingBoxVisibility => - Json.obj("name" -> "updateUserBoundingBoxVisibility", - "value" -> Json.toJson(s)(UpdateUserBoundingBoxVisibility.jsonFormat)) - case s: RemoveFallbackLayer => - Json.obj("name" -> "removeFallbackLayer", "value" -> Json.toJson(s)(RemoveFallbackLayer.jsonFormat)) - case s: ImportVolumeData => - Json.obj("name" -> "importVolumeTracing", "value" -> Json.toJson(s)(ImportVolumeData.jsonFormat)) - case s: UpdateTdCamera => - Json.obj("name" -> "updateTdCamera", "value" -> Json.toJson(s)(UpdateTdCamera.jsonFormat)) - case s: CreateSegmentVolumeAction => - Json.obj("name" -> "createSegment", "value" -> Json.toJson(s)(CreateSegmentVolumeAction.jsonFormat)) - case s: UpdateSegmentVolumeAction => - Json.obj("name" -> "updateSegment", "value" -> Json.toJson(s)(UpdateSegmentVolumeAction.jsonFormat)) - case s: DeleteSegmentVolumeAction => - Json.obj("name" -> "deleteSegment", "value" -> Json.toJson(s)(DeleteSegmentVolumeAction.jsonFormat)) - case s: UpdateSegmentGroupsVolumeAction => - Json.obj("name" -> "updateSegmentGroups", "value" -> Json.toJson(s)(UpdateSegmentGroupsVolumeAction.jsonFormat)) - case s: CompactVolumeUpdateAction => Json.toJson(s)(CompactVolumeUpdateAction.compactVolumeUpdateActionFormat) - } - } - -} From 21d1078b64c9e984768bfae0168466816661fd54 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 17 Jul 2024 11:46:59 +0200 Subject: [PATCH 022/150] further unify update actions, add tracingId to LayerUpdateActions --- .../annotation/UpdateActions.scala | 32 +++++++++-- .../controllers/VolumeTracingController.scala | 32 +++++------ .../EditableMappingService.scala | 5 +- .../skeleton/SkeletonTracingService.scala | 8 +-- .../updating/SkeletonUpdateActions.scala | 41 +++++++++---- .../volume/VolumeTracingService.scala | 20 +++---- .../tracings/volume/VolumeUpdateActions.scala | 57 +++++++++++++------ 7 files changed, 128 insertions(+), 67 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala index 1858251c968..04c58bf3235 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala @@ -55,13 +55,17 @@ trait UpdateAction { def isViewOnlyChange: Boolean = false } +trait LayerUpdateAction extends UpdateAction { + def actionTracingId: String +} + object UpdateAction { implicit object updateActionFormat extends Format[UpdateAction] { override def reads(json: JsValue): JsResult[UpdateAction] = { val jsonValue = (json \ "value").as[JsObject] (json \ "name").as[String] match { - // Skeletons + // Skeleton case "createTree" => deserialize[CreateTreeSkeletonAction](jsonValue) case "deleteTree" => deserialize[DeleteTreeSkeletonAction](jsonValue) case "updateTree" => deserialize[UpdateTreeSkeletonAction](jsonValue) @@ -81,7 +85,7 @@ object UpdateAction { case "updateUserBoundingBoxVisibility" => deserialize[UpdateUserBoundingBoxVisibilitySkeletonAction](jsonValue) - // Volumes + // Volume case "updateBucket" => deserialize[UpdateBucketVolumeAction](jsonValue) case "updateVolumeTracing" => deserialize[UpdateTracingVolumeAction](jsonValue) case "updateUserBoundingBoxes" => deserialize[UpdateUserBoundingBoxesVolumeAction](jsonValue) @@ -96,11 +100,15 @@ object UpdateAction { case "deleteSegmentData" => deserialize[DeleteSegmentDataVolumeAction](jsonValue) case "updateMappingName" => deserialize[UpdateMappingNameVolumeAction](jsonValue) - // Editable Mappings + // Editable Mapping case "mergeAgglomerate" => deserialize[MergeAgglomerateUpdateAction](jsonValue) case "splitAgglomerate" => deserialize[SplitAgglomerateUpdateAction](jsonValue) - // TODO: Annotation, RevertToVersion + // Annotation + case "addLayerToAnnotation" => deserialize[AddLayerAnnotationUpdateAction](jsonValue) + case "deleteLayerFromAnnotation" => deserialize[DeleteLayerAnnotationUpdateAction](jsonValue) + case "updateLayerMetadata" => deserialize[UpdateLayerMetadataAnnotationUpdateAction](jsonValue) + case "updateMetadataOfAnnotation" => deserialize[UpdateMetadataAnnotationUpdateAction](jsonValue) case unknownAction: String => JsError(s"Invalid update action s'$unknownAction'") } @@ -117,6 +125,7 @@ object UpdateAction { (JsPath \ "position").json.update(JsPath.read[List[Float]].map(position => Json.toJson(position.map(_.toInt)))) override def writes(a: UpdateAction): JsValue = a match { + // Skeleton case s: CreateTreeSkeletonAction => Json.obj("name" -> "createTree", "value" -> Json.toJson(s)(CreateTreeSkeletonAction.jsonFormat)) case s: DeleteTreeSkeletonAction => @@ -161,6 +170,7 @@ object UpdateAction { case s: UpdateTdCameraSkeletonAction => Json.obj("name" -> "updateTdCamera", "value" -> Json.toJson(s)(UpdateTdCameraSkeletonAction.jsonFormat)) + // Volume case s: UpdateBucketVolumeAction => Json.obj("name" -> "updateBucket", "value" -> Json.toJson(s)(UpdateBucketVolumeAction.jsonFormat)) case s: UpdateTracingVolumeAction => @@ -187,10 +197,24 @@ object UpdateAction { Json.obj("name" -> "updateSegmentGroups", "value" -> Json.toJson(s)(UpdateSegmentGroupsVolumeAction.jsonFormat)) case s: CompactVolumeUpdateAction => Json.toJson(s)(CompactVolumeUpdateAction.compactVolumeUpdateActionFormat) + // Editable Mapping case s: SplitAgglomerateUpdateAction => Json.obj("name" -> "splitAgglomerate", "value" -> Json.toJson(s)(SplitAgglomerateUpdateAction.jsonFormat)) case s: MergeAgglomerateUpdateAction => Json.obj("name" -> "mergeAgglomerate", "value" -> Json.toJson(s)(MergeAgglomerateUpdateAction.jsonFormat)) + + // Annotation + case s: AddLayerAnnotationUpdateAction => + Json.obj("name" -> "addLayerToAnnotation", "value" -> Json.toJson(s)(AddLayerAnnotationUpdateAction.jsonFormat)) + case s: DeleteLayerAnnotationUpdateAction => + Json.obj("name" -> "deleteLayerFromAnnotation", + "value" -> Json.toJson(s)(DeleteLayerAnnotationUpdateAction.jsonFormat)) + case s: UpdateLayerMetadataAnnotationUpdateAction => + Json.obj("name" -> "updateLayerMetadata", + "value" -> Json.toJson(s)(UpdateLayerMetadataAnnotationUpdateAction.jsonFormat)) + case s: UpdateMetadataAnnotationUpdateAction => + Json.obj("name" -> "updateMetadataOfAnnotation", + "value" -> Json.toJson(s)(UpdateMetadataAnnotationUpdateAction.jsonFormat)) } } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index dcfd07b026d..10acdac3bb8 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -26,10 +26,10 @@ import com.scalableminds.webknossos.datastore.services.{ FullMeshRequest, UserAccessRequest } +import com.scalableminds.webknossos.tracingstore.annotation.UpdateActionGroup import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ EditableMappingService, - EditableMappingUpdateActionGroup, MinCutParameters, NeighborsParameters } @@ -43,7 +43,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ VolumeSegmentStatisticsService, VolumeTracingService } -import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, UpdateActionGroup} +import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits} import com.scalableminds.webknossos.tracingstore.{ TSRemoteDatastoreClient, TSRemoteWebknossosClient, @@ -369,20 +369,20 @@ class VolumeTracingController @Inject()( (editableMappingId, editableMappingInfo) <- editableMappingService.create( baseMappingName = tracingMappingName) volumeUpdate = UpdateMappingNameVolumeAction(Some(editableMappingId), - isEditable = Some(true), - isLocked = Some(true), - actionTimestamp = Some(System.currentTimeMillis())) + isEditable = Some(true), + isLocked = Some(true), + actionTimestamp = Some(System.currentTimeMillis())) _ <- tracingService.handleUpdateGroup( tracingId, - UpdateActionGroup[VolumeTracing](tracing.version + 1, - System.currentTimeMillis(), - None, - List(volumeUpdate), - None, - None, - "dummyTransactionId", - 1, - 0), + UpdateActionGroup(tracing.version + 1, + System.currentTimeMillis(), + None, + List(volumeUpdate), + None, + None, + "dummyTransactionId", + 1, + 0), tracing.version, urlOrHeaderToken(token, request) ) @@ -428,8 +428,8 @@ class VolumeTracingController @Inject()( } } - def updateEditableMapping(token: Option[String], tracingId: String): Action[List[EditableMappingUpdateActionGroup]] = - Action.async(validateJson[List[EditableMappingUpdateActionGroup]]) { implicit request => + def updateEditableMapping(token: Option[String], tracingId: String): Action[List[UpdateActionGroup]] = + Action.async(validateJson[List[UpdateActionGroup]]) { implicit request => accessTokenService.validateAccess(UserAccessRequest.writeTracing(tracingId), urlOrHeaderToken(token, request)) { for { tracing <- tracingService.find(tracingId) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index ff66dd20ac2..672814aff9d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -21,6 +21,7 @@ import com.scalableminds.webknossos.datastore.services.{ AdHocMeshServiceHolder, BinaryDataService } +import com.scalableminds.webknossos.tracingstore.annotation.UpdateActionGroup import com.scalableminds.webknossos.tracingstore.tracings.volume.ReversionHelper import com.scalableminds.webknossos.tracingstore.tracings.{ FallbackDataHelper, @@ -264,9 +265,7 @@ class EditableMappingService @Inject()( _ => applyPendingUpdates(editableMappingId, desiredVersion, remoteFallbackLayer, userToken)) } yield (materializedInfo, desiredVersion) - def update(editableMappingId: String, - updateActionGroup: EditableMappingUpdateActionGroup, - newVersion: Long): Fox[Unit] = + def update(editableMappingId: String, updateActionGroup: UpdateActionGroup, newVersion: Long): Fox[Unit] = for { actionsWithTimestamp <- Fox.successful(updateActionGroup.actions.map(_.addTimestamp(updateActionGroup.timestamp))) _ <- tracingDataStore.editableMappingUpdates.put(editableMappingId, newVersion, actionsWithTimestamp) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala index 5be705493fa..5eb19317295 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala @@ -8,6 +8,7 @@ import com.scalableminds.webknossos.datastore.geometry.NamedBoundingBoxProto import com.scalableminds.webknossos.datastore.helpers.{ProtoGeometryImplicits, SkeletonTracingDefaults} import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis import com.scalableminds.webknossos.tracingstore.TracingStoreRedisStore +import com.scalableminds.webknossos.tracingstore.annotation.UpdateActionGroup import com.scalableminds.webknossos.tracingstore.tracings.UpdateAction.SkeletonUpdateAction import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating._ @@ -36,16 +37,13 @@ class SkeletonTracingService @Inject()( implicit val tracingCompanion: SkeletonTracing.type = SkeletonTracing - implicit val updateActionJsonFormat: SkeletonUpdateAction.skeletonUpdateActionFormat.type = - SkeletonUpdateAction.skeletonUpdateActionFormat - def currentVersion(tracingId: String): Fox[Long] = tracingDataStore.skeletonUpdates.getVersion(tracingId, mayBeEmpty = Some(true), emptyFallback = Some(0L)) def currentVersion(tracing: SkeletonTracing): Long = tracing.version def handleUpdateGroup(tracingId: String, - updateActionGroup: UpdateActionGroup[SkeletonTracing], + updateActionGroup: UpdateActionGroup, previousVersion: Long, userToken: Option[String]): Fox[_] = tracingDataStore.skeletonUpdates.put( @@ -118,7 +116,7 @@ class SkeletonTracingService @Inject()( case Full(tracing) => remainingUpdates match { case List() => Fox.successful(tracing) - case RevertToVersionSkeletonAction(sourceVersion, _, _, _) :: tail => + case RevertToVersionSkeletonAction(tracingId, sourceVersion, _, _, _) :: tail => val sourceTracing = find(tracingId, Some(sourceVersion), useCache = false, applyUpdates = true) updateIter(sourceTracing, tail) case update :: tail => updateIter(Full(update.applyOn(tracing)), tail) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala index e24c7d59f68..cf43dc92e47 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala @@ -4,11 +4,11 @@ import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.util.geometry.{Vec3Double, Vec3Int} import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate -import com.scalableminds.webknossos.tracingstore.annotation.UpdateAction +import com.scalableminds.webknossos.tracingstore.annotation.{LayerUpdateAction, UpdateAction} import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.TreeType.TreeType import play.api.libs.json._ -trait SkeletonUpdateAction extends UpdateAction +trait SkeletonUpdateAction extends LayerUpdateAction case class CreateTreeSkeletonAction(id: Int, color: Option[com.scalableminds.util.image.Color], @@ -18,11 +18,12 @@ case class CreateTreeSkeletonAction(id: Int, comments: List[UpdateActionComment], groupId: Option[Int], isVisible: Option[Boolean], + `type`: Option[TreeType] = None, + edgesAreVisible: Option[Boolean], + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, - info: Option[String] = None, - `type`: Option[TreeType] = None, - edgesAreVisible: Option[Boolean]) + info: Option[String] = None) extends SkeletonUpdateAction with SkeletonUpdateActionHelper { /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { @@ -51,6 +52,7 @@ case class CreateTreeSkeletonAction(id: Int, } case class DeleteTreeSkeletonAction(id: Int, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -74,10 +76,11 @@ case class UpdateTreeSkeletonAction(id: Int, branchPoints: List[UpdateActionBranchPoint], comments: List[UpdateActionComment], groupId: Option[Int], + `type`: Option[TreeType] = None, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, - info: Option[String] = None, - `type`: Option[TreeType] = None) + info: Option[String] = None) extends SkeletonUpdateAction with SkeletonUpdateActionHelper { /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { @@ -104,6 +107,7 @@ case class UpdateTreeSkeletonAction(id: Int, case class MergeTreeSkeletonAction(sourceId: Int, targetId: Int, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -136,6 +140,7 @@ case class MergeTreeSkeletonAction(sourceId: Int, case class MoveTreeComponentSkeletonAction(nodeIds: List[Int], sourceId: Int, targetId: Int, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -179,6 +184,7 @@ case class MoveTreeComponentSkeletonAction(nodeIds: List[Int], case class CreateEdgeSkeletonAction(source: Int, target: Int, treeId: Int, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -201,6 +207,7 @@ case class CreateEdgeSkeletonAction(source: Int, case class DeleteEdgeSkeletonAction(source: Int, target: Int, treeId: Int, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -230,10 +237,11 @@ case class CreateNodeSkeletonAction(id: Int, interpolation: Option[Boolean], treeId: Int, timestamp: Long, + additionalCoordinates: Option[Seq[AdditionalCoordinate]] = None, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, - info: Option[String] = None, - additionalCoordinates: Option[Seq[AdditionalCoordinate]] = None) + info: Option[String] = None) extends SkeletonUpdateAction with SkeletonUpdateActionHelper with ProtoGeometryImplicits { @@ -276,10 +284,11 @@ case class UpdateNodeSkeletonAction(id: Int, interpolation: Option[Boolean], treeId: Int, timestamp: Long, + additionalCoordinates: Option[Seq[AdditionalCoordinate]] = None, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, - info: Option[String] = None, - additionalCoordinates: Option[Seq[AdditionalCoordinate]] = None) + info: Option[String] = None) extends SkeletonUpdateAction with SkeletonUpdateActionHelper with ProtoGeometryImplicits { @@ -318,6 +327,7 @@ case class UpdateNodeSkeletonAction(id: Int, case class DeleteNodeSkeletonAction(nodeId: Int, treeId: Int, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -341,6 +351,7 @@ case class DeleteNodeSkeletonAction(nodeId: Int, } case class UpdateTreeGroupsSkeletonAction(treeGroups: List[UpdateActionTreeGroup], + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -363,6 +374,7 @@ case class UpdateTracingSkeletonAction(activeNode: Option[Int], editRotation: com.scalableminds.util.geometry.Vec3Double, zoomLevel: Double, userBoundingBox: Option[com.scalableminds.util.geometry.BoundingBox], + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None, @@ -389,6 +401,7 @@ case class UpdateTracingSkeletonAction(activeNode: Option[Int], } case class RevertToVersionSkeletonAction(sourceVersion: Long, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -407,6 +420,7 @@ case class RevertToVersionSkeletonAction(sourceVersion: Long, case class UpdateTreeVisibilitySkeletonAction(treeId: Int, isVisible: Boolean, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -429,6 +443,7 @@ case class UpdateTreeVisibilitySkeletonAction(treeId: Int, case class UpdateTreeGroupVisibilitySkeletonAction(treeGroupId: Option[Int], isVisible: Boolean, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -467,6 +482,7 @@ case class UpdateTreeGroupVisibilitySkeletonAction(treeGroupId: Option[Int], case class UpdateTreeEdgesVisibilitySkeletonAction(treeId: Int, edgesAreVisible: Boolean, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -489,6 +505,7 @@ case class UpdateTreeEdgesVisibilitySkeletonAction(treeId: Int, } case class UpdateUserBoundingBoxesSkeletonAction(boundingBoxes: List[NamedBoundingBox], + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -507,6 +524,7 @@ case class UpdateUserBoundingBoxesSkeletonAction(boundingBoxes: List[NamedBoundi case class UpdateUserBoundingBoxVisibilitySkeletonAction(boundingBoxId: Option[Int], isVisible: Boolean, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -534,6 +552,7 @@ case class UpdateUserBoundingBoxVisibilitySkeletonAction(boundingBoxId: Option[I case class UpdateTdCameraSkeletonAction(actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, + actionTracingId: String, info: Option[String] = None) extends SkeletonUpdateAction { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 7a19b438837..5732d209dc4 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -23,6 +23,7 @@ import com.scalableminds.webknossos.datastore.models.{ WebknossosAdHocMeshRequest } import com.scalableminds.webknossos.datastore.services._ +import com.scalableminds.webknossos.tracingstore.annotation.UpdateActionGroup import com.scalableminds.webknossos.tracingstore.tracings.TracingType.TracingType import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService @@ -77,9 +78,6 @@ class VolumeTracingService @Inject()( implicit val tracingCompanion: VolumeTracing.type = VolumeTracing - implicit val updateActionJsonFormat: VolumeUpdateAction.volumeUpdateActionFormat.type = - VolumeUpdateAction.volumeUpdateActionFormat - val tracingType: TracingType = TracingType.volume val tracingStore: FossilDBClient = tracingDataStore.volumes @@ -117,7 +115,7 @@ class VolumeTracingService @Inject()( editableMappingTracingId) ?~> "volumeSegmentIndex.update.failed" def handleUpdateGroup(tracingId: String, - updateGroup: UpdateActionGroup[VolumeTracing], + updateGroup: UpdateActionGroup, previousVersion: Long, userToken: Option[String]): Fox[Unit] = for { @@ -162,9 +160,9 @@ class VolumeTracingService @Inject()( Fox.failure("Cannot delete segment data for annotations without segment index.") } else deleteSegmentData(tracingId, tracing, a, segmentIndexBuffer, updateGroup.version, userToken) ?~> "Failed to delete segment data." - case _: UpdateTdCameraVolumeAction => Fox.successful(tracing) - case a: ApplyableVolumeAction => Fox.successful(a.applyOn(tracing)) - case _ => Fox.failure("Unknown action.") + case _: UpdateTdCameraVolumeAction => Fox.successful(tracing) + case a: ApplyableVolumeAction => Fox.successful(a.applyOn(tracing)) + case _ => Fox.failure("Unknown action.") } case Empty => Fox.empty @@ -905,11 +903,11 @@ class VolumeTracingService @Inject()( editableMappingTracingId(tracing, tracingId)) } _ <- Fox.runIf(!dryRun)(segmentIndexBuffer.flush()) - updateGroup = UpdateActionGroup[VolumeTracing]( + updateGroup = UpdateActionGroup( tracing.version + 1L, System.currentTimeMillis(), None, - List(AddSegmentIndexVolumeAction()), + List(AddSegmentIndexVolumeAction(tracingId)), None, None, "dummyTransactionId", @@ -992,11 +990,11 @@ class VolumeTracingService @Inject()( } yield () } _ <- segmentIndexBuffer.flush() - updateGroup = UpdateActionGroup[VolumeTracing]( + updateGroup = UpdateActionGroup( tracing.version + 1, System.currentTimeMillis(), None, - List(ImportVolumeDataVolumeAction(Some(mergedVolume.largestSegmentId.toPositiveLong))), + List(ImportVolumeDataVolumeAction(tracingId, Some(mergedVolume.largestSegmentId.toPositiveLong))), None, None, "dummyTransactionId", diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala index 360bd55c055..c36ade7c817 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala @@ -5,7 +5,7 @@ import com.scalableminds.util.geometry.{Vec3Double, Vec3Int} import com.scalableminds.webknossos.datastore.VolumeTracing.{Segment, SegmentGroup, VolumeTracing} import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate -import com.scalableminds.webknossos.tracingstore.annotation.UpdateAction +import com.scalableminds.webknossos.tracingstore.annotation.{LayerUpdateAction, UpdateAction} import com.scalableminds.webknossos.tracingstore.tracings.NamedBoundingBox import play.api.libs.json._ @@ -22,7 +22,7 @@ trait VolumeUpdateActionHelper { } -trait VolumeUpdateAction extends UpdateAction +trait VolumeUpdateAction extends LayerUpdateAction trait ApplyableVolumeAction extends VolumeUpdateAction @@ -30,10 +30,11 @@ case class UpdateBucketVolumeAction(position: Vec3Int, cubeSize: Int, mag: Vec3Int, base64Data: String, + additionalCoordinates: Option[Seq[AdditionalCoordinate]] = None, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, - info: Option[String] = None, - additionalCoordinates: Option[Seq[AdditionalCoordinate]] = None) + info: Option[String] = None) extends VolumeUpdateAction { lazy val data: Array[Byte] = Base64.getDecoder.decode(base64Data) @@ -43,7 +44,7 @@ case class UpdateBucketVolumeAction(position: Vec3Int, override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) def transformToCompact: CompactVolumeUpdateAction = - CompactVolumeUpdateAction("updateBucket", actionTimestamp, actionAuthorId, Json.obj()) + CompactVolumeUpdateAction("updateBucket", Json.obj(), actionTracingId, actionTimestamp, actionAuthorId, info) } case class UpdateTracingVolumeAction( @@ -52,10 +53,11 @@ case class UpdateTracingVolumeAction( editRotation: Vec3Double, largestSegmentId: Option[Long], zoomLevel: Double, + editPositionAdditionalCoordinates: Option[Seq[AdditionalCoordinate]] = None, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, - info: Option[String] = None, - editPositionAdditionalCoordinates: Option[Seq[AdditionalCoordinate]] = None + info: Option[String] = None ) extends VolumeUpdateAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = @@ -66,6 +68,7 @@ case class UpdateTracingVolumeAction( } case class RevertToVersionVolumeAction(sourceVersion: Long, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -78,6 +81,7 @@ case class RevertToVersionVolumeAction(sourceVersion: Long, } case class UpdateUserBoundingBoxesVolumeAction(boundingBoxes: List[NamedBoundingBox], + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -94,6 +98,7 @@ case class UpdateUserBoundingBoxesVolumeAction(boundingBoxes: List[NamedBounding case class UpdateUserBoundingBoxVisibilityVolumeAction(boundingBoxId: Option[Int], isVisible: Boolean, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -119,7 +124,8 @@ case class UpdateUserBoundingBoxVisibilityVolumeAction(boundingBoxId: Option[Int }*/ } -case class RemoveFallbackLayerVolumeAction(actionTimestamp: Option[Long] = None, +case class RemoveFallbackLayerVolumeAction(actionTracingId: String, + actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) extends ApplyableVolumeAction { @@ -132,7 +138,8 @@ case class RemoveFallbackLayerVolumeAction(actionTimestamp: Option[Long] = None, tracing.clearFallbackLayer*/ } -case class ImportVolumeDataVolumeAction(largestSegmentId: Option[Long], +case class ImportVolumeDataVolumeAction(actionTracingId: String, + largestSegmentId: Option[Long], actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -146,7 +153,8 @@ case class ImportVolumeDataVolumeAction(largestSegmentId: Option[Long], tracing.copy(largestSegmentId = largestSegmentId)*/ } -case class AddSegmentIndexVolumeAction(actionTimestamp: Option[Long] = None, +case class AddSegmentIndexVolumeAction(actionTracingId: String, + actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) extends ApplyableVolumeAction { @@ -160,7 +168,8 @@ case class AddSegmentIndexVolumeAction(actionTimestamp: Option[Long] = None, } -case class UpdateTdCameraVolumeAction(actionTimestamp: Option[Long] = None, +case class UpdateTdCameraVolumeAction(actionTracingId: String, + actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) extends VolumeUpdateAction { @@ -180,9 +189,10 @@ case class CreateSegmentVolumeAction(id: Long, color: Option[com.scalableminds.util.image.Color], groupId: Option[Int], creationTime: Option[Long], + additionalCoordinates: Option[Seq[AdditionalCoordinate]] = None, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, - additionalCoordinates: Option[Seq[AdditionalCoordinate]] = None, info: Option[String] = None) extends ApplyableVolumeAction with ProtoGeometryImplicits { @@ -212,9 +222,10 @@ case class UpdateSegmentVolumeAction(id: Long, color: Option[com.scalableminds.util.image.Color], creationTime: Option[Long], groupId: Option[Int], + additionalCoordinates: Option[Seq[AdditionalCoordinate]] = None, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, - additionalCoordinates: Option[Seq[AdditionalCoordinate]] = None, info: Option[String] = None) extends ApplyableVolumeAction with ProtoGeometryImplicits @@ -241,6 +252,7 @@ case class UpdateSegmentVolumeAction(id: Long, } case class DeleteSegmentVolumeAction(id: Long, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -258,6 +270,7 @@ case class DeleteSegmentVolumeAction(id: Long, } case class DeleteSegmentDataVolumeAction(id: Long, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -271,6 +284,7 @@ case class DeleteSegmentDataVolumeAction(id: Long, case class UpdateMappingNameVolumeAction(mappingName: Option[String], isEditable: Option[Boolean], isLocked: Option[Boolean], + actionTracingId: String, actionTimestamp: Option[Long], actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -289,6 +303,7 @@ case class UpdateMappingNameVolumeAction(mappingName: Option[String], } case class UpdateSegmentGroupsVolumeAction(segmentGroups: List[UpdateActionSegmentGroup], + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -304,9 +319,10 @@ case class UpdateSegmentGroupsVolumeAction(segmentGroups: List[UpdateActionSegme } case class CompactVolumeUpdateAction(name: String, + value: JsObject, + actionTracingId: String, actionTimestamp: Option[Long], actionAuthorId: Option[String] = None, - value: JsObject, info: Option[String] = None) extends VolumeUpdateAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -320,16 +336,23 @@ object CompactVolumeUpdateAction { override def reads(json: JsValue): JsResult[CompactVolumeUpdateAction] = for { name <- (json \ "name").validate[String] + actionTracingId <- (json \ "value" \ "actionTracingId").validate[String] actionTimestamp <- (json \ "value" \ "actionTimestamp").validateOpt[Long] actionAuthorId <- (json \ "value" \ "actionAuthorId").validateOpt[String] info <- (json \ "value" \ "info").validateOpt[String] value <- (json \ "value") .validate[JsObject] - .map(_ - "actionTimestamp") // TODO also separate out info + actionAuthorId - } yield CompactVolumeUpdateAction(name, actionTimestamp, actionAuthorId, value, info) + .map(_ - "actionTimestamp" - "actionTimestamp" - "actionAuthorId" - "info") + } yield CompactVolumeUpdateAction(name, value, actionTracingId, actionTimestamp, actionAuthorId, info) override def writes(o: CompactVolumeUpdateAction): JsValue = - Json.obj("name" -> o.name, "value" -> (Json.obj("actionTimestamp" -> o.actionTimestamp) ++ o.value)) + Json.obj( + "name" -> o.name, + "value" -> (Json.obj("actionTracingId" -> o.actionTracingId, + "actionTimestamp" -> o.actionTimestamp, + "actionAuthorId" -> o.actionAuthorId, + "info" -> o.info) ++ o.value) + ) } } From 24e8fd210c2e715d0297e8900fc3085380e6762b Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 17 Jul 2024 14:41:00 +0200 Subject: [PATCH 023/150] remove unused stuff, log routes, access checks --- app/controllers/UserTokenController.scala | 47 ++++++------ .../services/AccessTokenService.scala | 3 + .../AnnotationTransactionService.scala | 2 +- .../annotation/AnnotationUpdateActions.scala | 16 +++++ ...ervice.scala => TSAnnotationService.scala} | 57 ++++++++++++++- .../SkeletonTracingController.scala | 27 ------- ...ler.scala => TSAnnotationController.scala} | 38 ++++++++-- .../controllers/TracingController.scala | 66 ++++------------- .../controllers/VolumeTracingController.scala | 15 +--- .../tracings/TracingService.scala | 16 ++--- .../EditableMappingService.scala | 43 ++++------- .../EditableMappingUpdater.scala | 8 ++- .../skeleton/SkeletonTracingService.scala | 72 ++----------------- .../volume/VolumeTracingService.scala | 11 +-- ...alableminds.webknossos.tracingstore.routes | 10 ++- 15 files changed, 191 insertions(+), 240 deletions(-) rename webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/{DSAnnotationService.scala => TSAnnotationService.scala} (62%) rename webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/{DSAnnotationController.scala => TSAnnotationController.scala} (62%) diff --git a/app/controllers/UserTokenController.scala b/app/controllers/UserTokenController.scala index 410850dd38b..45ca19ae3fe 100644 --- a/app/controllers/UserTokenController.scala +++ b/app/controllers/UserTokenController.scala @@ -98,6 +98,8 @@ class UserTokenController @Inject()(datasetDAO: DatasetDAO, handleDataSourceAccess(accessRequest.resourceId, accessRequest.mode, userBox)(sharingTokenAccessCtx) case AccessResourceType.tracing => handleTracingAccess(accessRequest.resourceId.name, accessRequest.mode, userBox, token) + case AccessResourceType.annotation => + handleAnnotationAccess(accessRequest.resourceId.name, accessRequest.mode, userBox, token) case AccessResourceType.jobExport => handleJobExportAccess(accessRequest.resourceId.name, accessRequest.mode, userBox) case _ => @@ -160,7 +162,19 @@ class UserTokenController @Inject()(datasetDAO: DatasetDAO, private def handleTracingAccess(tracingId: String, mode: AccessMode, userBox: Box[User], - token: Option[String]): Fox[UserAccessAnswer] = { + token: Option[String]): Fox[UserAccessAnswer] = + if (tracingId == TracingIds.dummyTracingId) + Fox.successful(UserAccessAnswer(granted = true)) + else + for { + annotation <- annotationInformationProvider.annotationForTracing(tracingId)(GlobalAccessContext) ?~> "annotation.notFound" + result <- handleAnnotationAccess(annotation._id.toString, mode, userBox, token) + } yield result + + private def handleAnnotationAccess(annotationId: String, + mode: AccessMode, + userBox: Box[User], + token: Option[String]): Fox[UserAccessAnswer] = { // Access is explicitly checked by userBox, not by DBAccessContext, as there is no token sharing for annotations // Optionally, an accessToken can be provided which explicitly looks up the read right the private link table @@ -171,25 +185,18 @@ class UserTokenController @Inject()(datasetDAO: DatasetDAO, case _ => Fox.successful(false) } - if (tracingId == TracingIds.dummyTracingId) - Fox.successful(UserAccessAnswer(granted = true)) - else { - for { - annotation <- annotationInformationProvider.annotationForTracing(tracingId)(GlobalAccessContext) ?~> "annotation.notFound" - annotationAccessByToken <- token - .map(annotationPrivateLinkDAO.findOneByAccessToken) - .getOrElse(Fox.empty) - .futureBox - - allowedByToken = annotationAccessByToken.exists(annotation._id == _._annotation) - restrictions <- annotationInformationProvider.restrictionsFor( - AnnotationIdentifier(annotation.typ, annotation._id))(GlobalAccessContext) ?~> "restrictions.notFound" - allowedByUser <- checkRestrictions(restrictions) ?~> "restrictions.failedToCheck" - allowed = allowedByToken || allowedByUser - } yield { - if (allowed) UserAccessAnswer(granted = true) - else UserAccessAnswer(granted = false, Some(s"No ${mode.toString} access to tracing")) - } + // TODO is a dummy annotation id needed? + for { + annotation <- annotationInformationProvider.provideAnnotation(annotationId, userBox)(GlobalAccessContext) ?~> "annotation.notFound" + annotationAccessByToken <- token.map(annotationPrivateLinkDAO.findOneByAccessToken).getOrElse(Fox.empty).futureBox + allowedByToken = annotationAccessByToken.exists(annotation._id == _._annotation) + restrictions <- annotationInformationProvider.restrictionsFor( + AnnotationIdentifier(annotation.typ, annotation._id))(GlobalAccessContext) ?~> "restrictions.notFound" + allowedByUser <- checkRestrictions(restrictions) ?~> "restrictions.failedToCheck" + allowed = allowedByToken || allowedByUser + } yield { + if (allowed) UserAccessAnswer(granted = true) + else UserAccessAnswer(granted = false, Some(s"No ${mode.toString} access to tracing")) } } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AccessTokenService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AccessTokenService.scala index b21c415b13c..c7a593414fb 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AccessTokenService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AccessTokenService.scala @@ -46,6 +46,9 @@ object UserAccessRequest { def writeTracing(tracingId: String): UserAccessRequest = UserAccessRequest(DataSourceId(tracingId, ""), AccessResourceType.tracing, AccessMode.write) + def readAnnotation(annotationId: String): UserAccessRequest = + UserAccessRequest(DataSourceId(annotationId, ""), AccessResourceType.annotation, AccessMode.read) + def writeAnnotation(annotationId: String): UserAccessRequest = UserAccessRequest(DataSourceId(annotationId, ""), AccessResourceType.annotation, AccessMode.write) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala index a29f90e1f0a..3c7f5f48354 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala @@ -13,7 +13,7 @@ import scala.concurrent.duration._ class AnnotationTransactionService @Inject()( handledGroupIdStore: TracingStoreRedisStore, // TODO: instantiate here rather than with injection, give fix namespace prefix? uncommittedUpdatesStore: TracingStoreRedisStore, - annotationService: DSAnnotationService) { + annotationService: TSAnnotationService) { private val transactionGroupExpiry: FiniteDuration = 24 hours private val handledGroupCacheExpiry: FiniteDuration = 24 hours diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala index 725634f66e8..a00e21e9a7c 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala @@ -59,6 +59,18 @@ case class UpdateMetadataAnnotationUpdateAction(name: Option[String], this.copy(actionAuthorId = authorId) } +case class RevertToVersionUpdateAction(sourceVersion: Long, + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) + extends AnnotationUpdateAction { + override def addTimestamp(timestamp: Long): UpdateAction = + this.copy(actionTimestamp = Some(timestamp)) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def addAuthorId(authorId: Option[String]): UpdateAction = + this.copy(actionAuthorId = authorId) +} + object AddLayerAnnotationUpdateAction { implicit val jsonFormat: OFormat[AddLayerAnnotationUpdateAction] = Json.format[AddLayerAnnotationUpdateAction] } @@ -73,3 +85,7 @@ object UpdateMetadataAnnotationUpdateAction { implicit val jsonFormat: OFormat[UpdateMetadataAnnotationUpdateAction] = Json.format[UpdateMetadataAnnotationUpdateAction] } +object RevertToVersionUpdateAction { + implicit val jsonFormat: OFormat[RevertToVersionUpdateAction] = + Json.format[RevertToVersionUpdateAction] +} diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala similarity index 62% rename from webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala rename to webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index c0c634e5f74..ef5fe9ab81c 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/DSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -10,15 +10,21 @@ import com.scalableminds.webknossos.datastore.Annotation.{ UpdateLayerMetadataAnnotationUpdateAction, UpdateMetadataAnnotationUpdateAction } +import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{ + CreateNodeSkeletonAction, + DeleteNodeSkeletonAction, + UpdateTracingSkeletonAction +} import com.scalableminds.webknossos.tracingstore.tracings.volume.UpdateBucketVolumeAction import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingUpdatesReport} +import play.api.libs.json.{JsObject, JsValue, Json} import scalapb.GeneratedMessage import javax.inject.Inject import scala.concurrent.ExecutionContext -class DSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosClient, +class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosClient, tracingDataStore: TracingDataStore) extends KeyValueStoreImplicits { @@ -59,6 +65,18 @@ class DSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl } } + private def findPendingUpdates(annotationId: String, existingVersion: Long, desiredVersion: Long)( + implicit ec: ExecutionContext): Fox[List[UpdateAction]] = + if (desiredVersion == existingVersion) Fox.successful(List()) + else { + for { + updateActionGroups <- tracingDataStore.annotationUpdates.getMultipleVersions( + annotationId, + Some(desiredVersion), + Some(existingVersion + 1))(fromJsonBytes[List[UpdateAction]]) + } yield updateActionGroups.reverse.flatten + } + def applyUpdate(annotation: AnnotationProto, updateAction: GeneratedMessage)( implicit ec: ExecutionContext): Fox[AnnotationProto] = for { @@ -78,4 +96,41 @@ class DSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl } } yield withAppliedChange.copy(version = withAppliedChange.version + 1L) + def updateActionLog(annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]): Fox[JsValue] = { + def versionedTupleToJson(tuple: (Long, List[UpdateAction])): JsObject = + Json.obj( + "version" -> tuple._1, + "value" -> Json.toJson(tuple._2) + ) + + for { + updateActionGroups <- tracingDataStore.annotationUpdates.getMultipleVersionsAsVersionValueTuple( + annotationId, + newestVersion, + oldestVersion)(fromJsonBytes[List[UpdateAction]]) + updateActionGroupsJs = updateActionGroups.map(versionedTupleToJson) + } yield Json.toJson(updateActionGroupsJs) + } + + def updateActionStatistics(tracingId: String): Fox[JsObject] = + for { + updateActionGroups <- tracingDataStore.skeletonUpdates.getMultipleVersions(tracingId)( + fromJsonBytes[List[UpdateAction]]) + updateActions = updateActionGroups.flatten + } yield { + Json.obj( + "updateTracingActionCount" -> updateActions.count { + case _: UpdateTracingSkeletonAction => true + case _ => false + }, + "createNodeActionCount" -> updateActions.count { + case _: CreateNodeSkeletonAction => true + case _ => false + }, + "deleteNodeActionCount" -> updateActions.count { + case _: DeleteNodeSkeletonAction => true + case _ => false + } + ) + } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala index f56b6d0d26e..b2d0640788f 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala @@ -76,31 +76,4 @@ class SkeletonTracingController @Inject()(val tracingService: SkeletonTracingSer } } - def updateActionLog(token: Option[String], - tracingId: String, - newestVersion: Option[Long], - oldestVersion: Option[Long]): Action[AnyContent] = Action.async { implicit request => - log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { - for { - updateLog <- tracingService.updateActionLog(tracingId, newestVersion, oldestVersion) - } yield { - Ok(updateLog) - } - } - } - } - - def updateActionStatistics(token: Option[String], tracingId: String): Action[AnyContent] = Action.async { - implicit request => - log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { - for { - statistics <- tracingService.updateActionStatistics(tracingId) - } yield { - Ok(statistics) - } - } - } - } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala similarity index 62% rename from webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala rename to webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index d03ea14c69e..cf65236e3b5 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/DSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -6,21 +6,26 @@ import com.scalableminds.webknossos.datastore.controllers.Controller import com.scalableminds.webknossos.datastore.services.UserAccessRequest import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} import com.scalableminds.webknossos.tracingstore.TracingStoreAccessTokenService -import com.scalableminds.webknossos.tracingstore.annotation.{AnnotationTransactionService, UpdateActionGroup} +import com.scalableminds.webknossos.tracingstore.annotation.{ + AnnotationTransactionService, + TSAnnotationService, + UpdateActionGroup +} import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService import play.api.mvc.{Action, AnyContent, PlayBodyParsers} import scala.concurrent.ExecutionContext -class DSAnnotationController @Inject()( +class TSAnnotationController @Inject()( accessTokenService: TracingStoreAccessTokenService, slackNotificationService: TSSlackNotificationService, + annotationService: TSAnnotationService, annotationTransactionService: AnnotationTransactionService, tracingDataStore: TracingDataStore)(implicit ec: ExecutionContext, bodyParsers: PlayBodyParsers) extends Controller with KeyValueStoreImplicits { - def initialize(annotationId: String, token: Option[String]): Action[AnyContent] = + def initialize(token: Option[String], annotationId: String): Action[AnyContent] = Action.async { implicit request => log() { accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { @@ -31,7 +36,7 @@ class DSAnnotationController @Inject()( } } - def update(annotationId: String, token: Option[String]): Action[List[UpdateActionGroup]] = + def update(token: Option[String], annotationId: String): Action[List[UpdateActionGroup]] = Action.async(validateJson[List[UpdateActionGroup]]) { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { @@ -47,6 +52,31 @@ class DSAnnotationController @Inject()( } } + def updateActionLog(token: Option[String], + annotationId: String, + newestVersion: Option[Long] = None, + oldestVersion: Option[Long] = None): Action[AnyContent] = Action.async { implicit request => + log() { + accessTokenService.validateAccess(UserAccessRequest.readAnnotation(annotationId), + urlOrHeaderToken(token, request)) { + for { + updateLog <- annotationService.updateActionLog(annotationId, newestVersion, oldestVersion) + } yield Ok(updateLog) + } + } + } + + def updateActionStatistics(token: Option[String], tracingId: String): Action[AnyContent] = Action.async { + implicit request => + log() { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + for { + statistics <- annotationService.updateActionStatistics(tracingId) + } yield Ok(statistics) + } + } + } + } // get version history diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala index 4644d58e6a1..bad8308b096 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala @@ -1,25 +1,16 @@ package com.scalableminds.webknossos.tracingstore.controllers -import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox import com.scalableminds.util.tools.JsonHelper.{boxFormat, optionFormat} import com.scalableminds.webknossos.datastore.controllers.Controller import com.scalableminds.webknossos.datastore.services.UserAccessRequest +import com.scalableminds.webknossos.tracingstore.annotation.UpdateActionGroup import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService -import com.scalableminds.webknossos.tracingstore.tracings.{ - TracingSelector, - TracingService, - UpdateAction, - UpdateActionGroup -} -import com.scalableminds.webknossos.tracingstore.{ - TSRemoteWebknossosClient, - TracingStoreAccessTokenService, - TracingUpdatesReport -} +import com.scalableminds.webknossos.tracingstore.tracings.{TracingSelector, TracingService} +import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingStoreAccessTokenService} import net.liftweb.common.{Empty, Failure, Full} import play.api.i18n.Messages -import play.api.libs.json.{Format, Json} +import play.api.libs.json.Json import play.api.mvc.{Action, AnyContent, PlayBodyParsers} import scalapb.{GeneratedMessage, GeneratedMessageCompanion} @@ -46,8 +37,6 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C implicit def packMultipleOpt(tracings: List[Option[T]]): Ts - implicit val updateActionJsonFormat: Format[UpdateAction[T]] = tracingService.updateActionJsonFormat - implicit val ec: ExecutionContext implicit val bodyParsers: PlayBodyParsers @@ -121,8 +110,8 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C } } - def update(token: Option[String], tracingId: String): Action[List[UpdateActionGroup[T]]] = - Action.async(validateJson[List[UpdateActionGroup[T]]]) { implicit request => + def update(token: Option[String], tracingId: String): Action[List[UpdateActionGroup]] = + Action.async(validateJson[List[UpdateActionGroup]]) { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { accessTokenService.validateAccess(UserAccessRequest.writeTracing(tracingId), urlOrHeaderToken(token, request)) { @@ -148,7 +137,7 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C private def handleUpdateGroupForTransaction(tracingId: String, previousVersionFox: Fox[Long], - updateGroup: UpdateActionGroup[T], + updateGroup: UpdateActionGroup, userToken: Option[String]): Fox[Long] = for { previousCommittedVersion: Long <- previousVersionFox @@ -180,7 +169,7 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C // For an update group (that is the last of a transaction), fetch all previous uncommitted for the same transaction // and commit them all. private def commitWithPending(tracingId: String, - updateGroup: UpdateActionGroup[T], + updateGroup: UpdateActionGroup, userToken: Option[String]): Fox[Long] = for { previousActionGroupsToCommit <- tracingService.getAllUncommittedFor(tracingId, updateGroup.transactionId) @@ -192,12 +181,12 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C _ <- tracingService.removeAllUncommittedFor(tracingId, updateGroup.transactionId) } yield commitResult - private def concatenateUpdateGroupsOfTransaction(previousActionGroups: List[UpdateActionGroup[T]], - lastActionGroup: UpdateActionGroup[T]): UpdateActionGroup[T] = + private def concatenateUpdateGroupsOfTransaction(previousActionGroups: List[UpdateActionGroup], + lastActionGroup: UpdateActionGroup): UpdateActionGroup = if (previousActionGroups.isEmpty) lastActionGroup else { val allActionGroups = previousActionGroups :+ lastActionGroup - UpdateActionGroup[T]( + UpdateActionGroup( version = lastActionGroup.version, timestamp = lastActionGroup.timestamp, authorId = lastActionGroup.authorId, @@ -212,41 +201,14 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C // Perform version check and commit the passed updates private def commitUpdates(tracingId: String, - updateGroups: List[UpdateActionGroup[T]], - userToken: Option[String]): Fox[Long] = { - val currentCommittedVersion: Fox[Long] = tracingService.currentVersion(tracingId) - val report = TracingUpdatesReport( - tracingId, - timestamps = updateGroups.map(g => Instant(g.timestamp)), - statistics = updateGroups.flatMap(_.stats).lastOption, - significantChangesCount = updateGroups.map(_.significantChangesCount).sum, - viewChangesCount = updateGroups.map(_.viewChangesCount).sum, - userToken - ) - remoteWebknossosClient.reportTracingUpdates(report).flatMap { _ => - updateGroups.foldLeft(currentCommittedVersion) { (previousVersion, updateGroup) => - previousVersion.flatMap { prevVersion: Long => - if (prevVersion + 1 == updateGroup.version) { - tracingService - .handleUpdateGroup(tracingId, updateGroup, prevVersion, userToken) - .flatMap( - _ => - tracingService.saveToHandledGroupIdStore(tracingId, - updateGroup.transactionId, - updateGroup.version, - updateGroup.transactionGroupIndex)) - .map(_ => updateGroup.version) - } else failUnlessAlreadyHandled(updateGroup, tracingId, prevVersion) - } - } - } - } + updateGroups: List[UpdateActionGroup], + userToken: Option[String]): Fox[Long] = ??? /* If this update group has already been “handled” (successfully saved as either committed or uncommitted), * ignore it silently. This is in case the frontend sends a retry if it believes a save to be unsuccessful * despite the backend receiving it just fine. */ - private def failUnlessAlreadyHandled(updateGroup: UpdateActionGroup[T], + private def failUnlessAlreadyHandled(updateGroup: UpdateActionGroup, tracingId: String, previousVersion: Long): Fox[Long] = { val errorMessage = s"Incorrect version. Expected: ${previousVersion + 1}; Got: ${updateGroup.version}" diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 10acdac3bb8..0c733f0ed9a 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -371,6 +371,7 @@ class VolumeTracingController @Inject()( volumeUpdate = UpdateMappingNameVolumeAction(Some(editableMappingId), isEditable = Some(true), isLocked = Some(true), + actionTracingId = tracingId, actionTimestamp = Some(System.currentTimeMillis())) _ <- tracingService.handleUpdateGroup( tracingId, @@ -453,20 +454,6 @@ class VolumeTracingController @Inject()( } } - def editableMappingUpdateActionLog(token: Option[String], tracingId: String): Action[AnyContent] = Action.async { - implicit request => - log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { - for { - tracing <- tracingService.find(tracingId) - mappingName <- tracing.mappingName.toFox - _ <- bool2Fox(tracing.getMappingIsEditable) ?~> "Mapping is not editable" - updateLog <- editableMappingService.updateActionLog(mappingName) - } yield Ok(updateLog) - } - } - } - def editableMappingInfo(token: Option[String], tracingId: String, version: Option[Long]): Action[AnyContent] = Action.async { implicit request => log() { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala index 40bc6e1a123..7eacf130c2a 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala @@ -2,6 +2,7 @@ package com.scalableminds.webknossos.tracingstore.tracings import com.scalableminds.util.tools.{Fox, FoxImplicits, JsonHelper} import com.scalableminds.webknossos.tracingstore.TracingStoreRedisStore +import com.scalableminds.webknossos.tracingstore.annotation.UpdateActionGroup import com.scalableminds.webknossos.tracingstore.tracings.TracingType.TracingType import com.scalableminds.webknossos.tracingstore.tracings.volume.MergedVolumeStats import com.typesafe.scalalogging.LazyLogging @@ -46,8 +47,6 @@ trait TracingService[T <: GeneratedMessage] implicit def tracingCompanion: GeneratedMessageCompanion[T] - implicit val updateActionJsonFormat: Format[UpdateAction[T]] - // this should be longer than maxCacheTime in webknossos/AnnotationStore // so that the references saved there remain valid throughout their life private val temporaryStoreTimeout = 70 minutes @@ -75,7 +74,7 @@ trait TracingService[T <: GeneratedMessage] transactionId: String, transactionGroupIndex: Int, version: Long, - updateGroup: UpdateActionGroup[T], + updateGroup: UpdateActionGroup, expiry: FiniteDuration): Fox[Unit] = for { _ <- Fox.runIf(transactionGroupIndex > 0)( @@ -90,11 +89,11 @@ trait TracingService[T <: GeneratedMessage] Some(expiry)) } yield () - def getAllUncommittedFor(tracingId: String, transactionId: String): Fox[List[UpdateActionGroup[T]]] = + def getAllUncommittedFor(tracingId: String, transactionId: String): Fox[List[UpdateActionGroup]] = for { raw: Seq[String] <- uncommittedUpdatesStore.findAllConditional(patternFor(tracingId, transactionId)) - parsed: Seq[UpdateActionGroup[T]] = raw.flatMap(itemAsString => - JsonHelper.jsResultToOpt(Json.parse(itemAsString).validate[UpdateActionGroup[T]])) + parsed: Seq[UpdateActionGroup] = raw.flatMap(itemAsString => + JsonHelper.jsResultToOpt(Json.parse(itemAsString).validate[UpdateActionGroup])) } yield parsed.toList.sortBy(_.transactionGroupIndex) def removeAllUncommittedFor(tracingId: String, transactionId: String): Fox[Unit] = @@ -109,11 +108,6 @@ trait TracingService[T <: GeneratedMessage] Fox.successful(tracing) } - def handleUpdateGroup(tracingId: String, - updateGroup: UpdateActionGroup[T], - previousVersion: Long, - userToken: Option[String]): Fox[_] - def applyPendingUpdates(tracing: T, tracingId: String, targetVersion: Option[Long]): Fox[T] = Fox.successful(tracing) def find(tracingId: String, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index 672814aff9d..a61c9cd116d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -21,7 +21,7 @@ import com.scalableminds.webknossos.datastore.services.{ AdHocMeshServiceHolder, BinaryDataService } -import com.scalableminds.webknossos.tracingstore.annotation.UpdateActionGroup +import com.scalableminds.webknossos.tracingstore.annotation.{UpdateAction, UpdateActionGroup} import com.scalableminds.webknossos.tracingstore.tracings.volume.ReversionHelper import com.scalableminds.webknossos.tracingstore.tracings.{ FallbackDataHelper, @@ -36,7 +36,7 @@ import net.liftweb.common.{Box, Empty, Failure, Full} import net.liftweb.common.Box.tryo import org.jgrapht.alg.flow.PushRelabelMFImpl import org.jgrapht.graph.{DefaultWeightedEdge, SimpleWeightedGraph} -import play.api.libs.json.{JsObject, JsValue, Json, OFormat} +import play.api.libs.json.{JsObject, Json, OFormat} import java.nio.file.Paths import java.util @@ -182,9 +182,8 @@ class EditableMappingService @Inject()( _ <- duplicateSegmentToAgglomerate(editableMappingId, newId, newVersion) _ <- duplicateAgglomerateToGraph(editableMappingId, newId, newVersion) updateActionsWithVersions <- getUpdateActionsWithVersions(editableMappingId, editableMappingInfoAndVersion._2, 0L) - _ <- Fox.serialCombined(updateActionsWithVersions) { - updateActionsWithVersion: (Long, List[EditableMappingUpdateAction]) => - tracingDataStore.editableMappingUpdates.put(newId, updateActionsWithVersion._1, updateActionsWithVersion._2) + _ <- Fox.serialCombined(updateActionsWithVersions) { updateActionsWithVersion: (Long, List[UpdateAction]) => + tracingDataStore.editableMappingUpdates.put(newId, updateActionsWithVersion._1, updateActionsWithVersion._2) } } yield newId @@ -222,20 +221,6 @@ class EditableMappingService @Inject()( } yield () } - def updateActionLog(editableMappingId: String): Fox[JsValue] = { - def versionedTupleToJson(tuple: (Long, List[EditableMappingUpdateAction])): JsObject = - Json.obj( - "version" -> tuple._1, - "value" -> Json.toJson(tuple._2) - ) - - for { - updates <- tracingDataStore.editableMappingUpdates.getMultipleVersionsAsVersionValueTuple(editableMappingId)( - fromJsonBytes[List[EditableMappingUpdateAction]]) - updateActionGroupsJs = updates.map(versionedTupleToJson) - } yield Json.toJson(updateActionGroupsJs) - } - def getInfo(editableMappingId: String, version: Option[Long] = None, remoteFallbackLayer: RemoteFallbackLayer, @@ -311,7 +296,7 @@ class EditableMappingService @Inject()( private def getPendingUpdates(editableMappingId: String, closestMaterializedVersion: Long, - closestMaterializableVersion: Long): Fox[List[EditableMappingUpdateAction]] = + closestMaterializableVersion: Long): Fox[List[UpdateAction]] = if (closestMaterializableVersion == closestMaterializedVersion) { Fox.successful(List.empty) } else { @@ -322,22 +307,20 @@ class EditableMappingService @Inject()( } yield updates.map(_._2).reverse.flatten } - private def getUpdateActionsWithVersions( - editableMappingId: String, - newestVersion: Long, - oldestVersion: Long): Fox[List[(Long, List[EditableMappingUpdateAction])]] = { + private def getUpdateActionsWithVersions(editableMappingId: String, + newestVersion: Long, + oldestVersion: Long): Fox[List[(Long, List[UpdateAction])]] = { val batchRanges = batchRangeInclusive(oldestVersion, newestVersion, batchSize = 100) for { updateActionBatches <- Fox.serialCombined(batchRanges.toList) { batchRange => val batchFrom = batchRange._1 val batchTo = batchRange._2 for { - res <- tracingDataStore.editableMappingUpdates - .getMultipleVersionsAsVersionValueTuple[List[EditableMappingUpdateAction]]( - editableMappingId, - Some(batchTo), - Some(batchFrom) - )(fromJsonBytes[List[EditableMappingUpdateAction]]) + res <- tracingDataStore.editableMappingUpdates.getMultipleVersionsAsVersionValueTuple[List[UpdateAction]]( + editableMappingId, + Some(batchTo), + Some(batchFrom) + )(fromJsonBytes[List[UpdateAction]]) } yield res } ?~> "Failed to fetch editable mapping update actions from fossilDB" flat = updateActionBatches.flatten diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index 89303d32040..6f6e0aa1d61 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -9,6 +9,7 @@ import com.scalableminds.webknossos.datastore.SegmentToAgglomerateProto.{ SegmentToAgglomerateChunkProto } import com.scalableminds.webknossos.tracingstore.TSRemoteDatastoreClient +import com.scalableminds.webknossos.tracingstore.annotation.{RevertToVersionUpdateAction, UpdateAction} import com.scalableminds.webknossos.tracingstore.tracings.volume.ReversionHelper import com.scalableminds.webknossos.tracingstore.tracings.{ KeyValueStoreImplicits, @@ -53,7 +54,7 @@ class EditableMappingUpdater( private val agglomerateToGraphBuffer: mutable.Map[String, (AgglomerateGraph, Boolean)] = new mutable.HashMap[String, (AgglomerateGraph, Boolean)]() - def applyUpdatesAndSave(existingEditabeMappingInfo: EditableMappingInfo, updates: List[EditableMappingUpdateAction])( + def applyUpdatesAndSave(existingEditabeMappingInfo: EditableMappingInfo, updates: List[UpdateAction])( implicit ec: ExecutionContext): Fox[EditableMappingInfo] = for { updatedEditableMappingInfo <- updateIter(Some(existingEditabeMappingInfo), updates) @@ -86,7 +87,7 @@ class EditableMappingUpdater( tracingDataStore.editableMappingsAgglomerateToGraph.put(key, newVersion, valueToFlush) } - private def updateIter(mappingFox: Fox[EditableMappingInfo], remainingUpdates: List[EditableMappingUpdateAction])( + private def updateIter(mappingFox: Fox[EditableMappingInfo], remainingUpdates: List[UpdateAction])( implicit ec: ExecutionContext): Fox[EditableMappingInfo] = mappingFox.futureBox.flatMap { case Empty => @@ -107,7 +108,7 @@ class EditableMappingUpdater( mappingFox } - private def applyOneUpdate(mapping: EditableMappingInfo, update: EditableMappingUpdateAction)( + private def applyOneUpdate(mapping: EditableMappingInfo, update: UpdateAction)( implicit ec: ExecutionContext): Fox[EditableMappingInfo] = update match { case splitAction: SplitAgglomerateUpdateAction => @@ -116,6 +117,7 @@ class EditableMappingUpdater( applyMergeAction(mapping, mergeAction) ?~> "Failed to apply merge action" case revertAction: RevertToVersionUpdateAction => revertToVersion(revertAction) ?~> "Failed to apply revert action" + case _ => Fox.failure("this is not an editable mapping update action!") } private def applySplitAction(editableMappingInfo: EditableMappingInfo, update: SplitAgglomerateUpdateAction)( diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala index 5eb19317295..a70eb859f7d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala @@ -8,14 +8,12 @@ import com.scalableminds.webknossos.datastore.geometry.NamedBoundingBoxProto import com.scalableminds.webknossos.datastore.helpers.{ProtoGeometryImplicits, SkeletonTracingDefaults} import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis import com.scalableminds.webknossos.tracingstore.TracingStoreRedisStore -import com.scalableminds.webknossos.tracingstore.annotation.UpdateActionGroup -import com.scalableminds.webknossos.tracingstore.tracings.UpdateAction.SkeletonUpdateAction +import com.scalableminds.webknossos.tracingstore.annotation.LayerUpdateAction import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating._ import com.scalableminds.webknossos.tracingstore.tracings.volume.MergedVolumeStats import net.liftweb.common.{Box, Empty, Full} import play.api.i18n.MessagesProvider -import play.api.libs.json.{JsObject, JsValue, Json} import scala.concurrent.ExecutionContext @@ -42,20 +40,6 @@ class SkeletonTracingService @Inject()( def currentVersion(tracing: SkeletonTracing): Long = tracing.version - def handleUpdateGroup(tracingId: String, - updateActionGroup: UpdateActionGroup, - previousVersion: Long, - userToken: Option[String]): Fox[_] = - tracingDataStore.skeletonUpdates.put( - tracingId, - updateActionGroup.version, - updateActionGroup.actions - .map(_.addTimestamp(updateActionGroup.timestamp).addAuthorId(updateActionGroup.authorId)) match { //to the first action in the group, attach the group's info - case Nil => Nil - case first :: rest => first.addInfo(updateActionGroup.info) :: rest - } - ) - override def applyPendingUpdates(tracing: SkeletonTracing, tracingId: String, desiredVersion: Option[Long]): Fox[SkeletonTracing] = { @@ -94,16 +78,9 @@ class SkeletonTracingService @Inject()( private def findPendingUpdates(tracingId: String, existingVersion: Long, - desiredVersion: Long): Fox[List[SkeletonUpdateAction]] = - if (desiredVersion == existingVersion) Fox.successful(List()) - else { - for { - updateActionGroups <- tracingDataStore.skeletonUpdates.getMultipleVersions( - tracingId, - Some(desiredVersion), - Some(existingVersion + 1))(fromJsonBytes[List[SkeletonUpdateAction]]) - } yield updateActionGroups.reverse.flatten - } + desiredVersion: Long): Fox[List[SkeletonUpdateAction]] = ??? + + private def applyUpdateOn(tracing: SkeletonTracing, update: LayerUpdateAction): SkeletonTracing = ??? private def update(tracing: SkeletonTracing, tracingId: String, @@ -116,10 +93,10 @@ class SkeletonTracingService @Inject()( case Full(tracing) => remainingUpdates match { case List() => Fox.successful(tracing) - case RevertToVersionSkeletonAction(tracingId, sourceVersion, _, _, _) :: tail => + case RevertToVersionSkeletonAction(sourceVersion, tracingId, _, _, _) :: tail => val sourceTracing = find(tracingId, Some(sourceVersion), useCache = false, applyUpdates = true) updateIter(sourceTracing, tail) - case update :: tail => updateIter(Full(update.applyOn(tracing)), tail) + case update :: tail => updateIter(Full(applyUpdateOn(tracing, update)), tail) } case _ => tracingFox } @@ -210,43 +187,6 @@ class SkeletonTracingService @Inject()( userToken: Option[String])(implicit mp: MessagesProvider): Fox[MergedVolumeStats] = Fox.successful(MergedVolumeStats.empty()) - def updateActionLog(tracingId: String, newestVersion: Option[Long], oldestVersion: Option[Long]): Fox[JsValue] = { - def versionedTupleToJson(tuple: (Long, List[SkeletonUpdateAction])): JsObject = - Json.obj( - "version" -> tuple._1, - "value" -> Json.toJson(tuple._2) - ) - for { - updateActionGroups <- tracingDataStore.skeletonUpdates.getMultipleVersionsAsVersionValueTuple( - tracingId, - newestVersion, - oldestVersion)(fromJsonBytes[List[SkeletonUpdateAction]]) - updateActionGroupsJs = updateActionGroups.map(versionedTupleToJson) - } yield Json.toJson(updateActionGroupsJs) - } - - def updateActionStatistics(tracingId: String): Fox[JsObject] = - for { - updateActionGroups <- tracingDataStore.skeletonUpdates.getMultipleVersions(tracingId)( - fromJsonBytes[List[SkeletonUpdateAction]]) - updateActions = updateActionGroups.flatten - } yield { - Json.obj( - "updateTracingActionCount" -> updateActions.count { - case _: UpdateTracingSkeletonAction => true - case _ => false - }, - "createNodeActionCount" -> updateActions.count { - case _: CreateNodeSkeletonAction => true - case _ => false - }, - "deleteNodeActionCount" -> updateActions.count { - case _: DeleteNodeSkeletonAction => true - case _ => false - } - ) - } - def dummyTracing: SkeletonTracing = SkeletonTracingDefaults.createInstance def mergeEditableMappings(tracingsWithIds: List[(SkeletonTracing, String)], userToken: Option[String]): Fox[String] = diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 5732d209dc4..53d507dbfd9 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -114,6 +114,8 @@ class VolumeTracingService @Inject()( mappingName, editableMappingTracingId) ?~> "volumeSegmentIndex.update.failed" + private def applyUpdateOn(tracing: VolumeTracing, update: ApplyableVolumeAction): VolumeTracing = ??? + def handleUpdateGroup(tracingId: String, updateGroup: UpdateActionGroup, previousVersion: Long, @@ -161,7 +163,7 @@ class VolumeTracingService @Inject()( } else deleteSegmentData(tracingId, tracing, a, segmentIndexBuffer, updateGroup.version, userToken) ?~> "Failed to delete segment data." case _: UpdateTdCameraVolumeAction => Fox.successful(tracing) - case a: ApplyableVolumeAction => Fox.successful(a.applyOn(tracing)) + case a: ApplyableVolumeAction => Fox.successful(applyUpdateOn(tracing, a)) case _ => Fox.failure("Unknown action.") } case Empty => @@ -172,10 +174,9 @@ class VolumeTracingService @Inject()( } _ <- segmentIndexBuffer.flush() _ <- save(updatedTracing.copy(version = updateGroup.version), Some(tracingId), updateGroup.version) - _ <- tracingDataStore.volumeUpdates.put( - tracingId, - updateGroup.version, - updateGroup.actions.map(_.addTimestamp(updateGroup.timestamp)).map(_.transformToCompact)) + _ <- tracingDataStore.volumeUpdates.put(tracingId, + updateGroup.version, + updateGroup.actions.map(_.addTimestamp(updateGroup.timestamp))) } yield Fox.successful(()) private def updateBucket(tracingId: String, diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index da0b1f93131..d4079a604d8 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -5,8 +5,10 @@ # Health endpoint GET /health @com.scalableminds.webknossos.tracingstore.controllers.Application.health -POST /annotation/initialize @com.scalableminds.webknossos.tracingstore.controllers.DSAnnotationController.initialize(annotationId: String, token: Option[String]) -POST /annotation/update @com.scalableminds.webknossos.tracingstore.controllers.DSAnnotationController.update(annotationId: String, token: Option[String]) +POST /annotation/initialize @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.initialize(token: Option[String], annotationId: String) +POST /annotation/update @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.update(token: Option[String], annotationId: String) +POST /annotation/:annotationId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionLog(token: Option[String], annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]) +GET /annotation/:annotationId/updateActionStatistics @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionStatistics(token: Option[String], annotationId: String) # Volume tracings POST /volume/save @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.save(token: Option[String]) @@ -18,7 +20,6 @@ POST /volume/:tracingId/update @com.scalablemin GET /volume/:tracingId/allDataZip @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.allDataZip(token: Option[String], tracingId: String, volumeDataZipFormat: String, version: Option[Long], voxelSize: Option[String], voxelSizeUnit: Option[String]) POST /volume/:tracingId/data @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.data(token: Option[String], tracingId: String) POST /volume/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.duplicate(token: Option[String], tracingId: String, fromTask: Option[Boolean], minResolution: Option[Int], maxResolution: Option[Int], downsample: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) -GET /volume/:tracingId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.updateActionLog(token: Option[String], tracingId: String, newestVersion: Option[Long], oldestVersion: Option[Long]) POST /volume/:tracingId/adHocMesh @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.requestAdHocMesh(token: Option[String], tracingId: String) POST /volume/:tracingId/fullMesh.stl @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.loadFullMeshStl(token: Option[String], tracingId: String) POST /volume/:tracingId/segmentIndex/:segmentId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentIndex(token: Option[String], tracingId: String, segmentId: Long) @@ -37,7 +38,6 @@ POST /volume/mergedFromContents @com.scalablemin # Editable Mappings POST /mapping/:tracingId/update @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.updateEditableMapping(token: Option[String], tracingId: String) -GET /mapping/:tracingId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingUpdateActionLog(token: Option[String], tracingId: String) GET /mapping/:tracingId/info @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingInfo(token: Option[String], tracingId: String, version: Option[Long]) GET /mapping/:tracingId/segmentsForAgglomerate @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingSegmentIdsForAgglomerate(token: Option[String], tracingId: String, agglomerateId: Long) GET /mapping/:tracingId/agglomerateIdForSegmentId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingAgglomerateIdForSegmentId(token: Option[String], tracingId: String, segmentId: Long) @@ -64,8 +64,6 @@ POST /skeleton/mergedFromIds @com.scalablemin GET /skeleton/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.get(token: Option[String], tracingId: String, version: Option[Long]) GET /skeleton/:tracingId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.newestVersion(token: Option[String], tracingId: String) -GET /skeleton/:tracingId/updateActionStatistics @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.updateActionStatistics(token: Option[String], tracingId: String) -GET /skeleton/:tracingId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.updateActionLog(token: Option[String], tracingId: String, newestVersion: Option[Long], oldestVersion: Option[Long]) POST /skeleton/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.getMultiple(token: Option[String]) POST /skeleton/:tracingId/update @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.update(token: Option[String], tracingId: String) From 2cbadf24754cc7956840d0d89ffa2ab7ef56c5d0 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 23 Jul 2024 14:09:40 +0200 Subject: [PATCH 024/150] fetch pending updates --- .../annotation/AnnotationLayerType.scala | 7 ++ .../annotation/AnnotationUpdateActions.scala | 4 +- .../annotation/TSAnnotationService.scala | 94 +++++++++++++++---- .../annotation/UpdateActions.scala | 7 +- .../controllers/TSAnnotationController.scala | 17 ++++ .../tracings/TracingDataStore.scala | 4 +- ...alableminds.webknossos.tracingstore.routes | 3 +- 7 files changed, 112 insertions(+), 24 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayerType.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayerType.scala index 0a9576b91aa..756180cbbd8 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayerType.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayerType.scala @@ -1,8 +1,15 @@ package com.scalableminds.webknossos.datastore.models.annotation import com.scalableminds.util.enumeration.ExtendedEnumeration +import com.scalableminds.webknossos.datastore.Annotation.AnnotationLayerTypeProto object AnnotationLayerType extends ExtendedEnumeration { type AnnotationLayerType = Value val Skeleton, Volume = Value + + def toProto(annotationLayerType: AnnotationLayerType): AnnotationLayerTypeProto = + annotationLayerType match { + case Skeleton => AnnotationLayerTypeProto.skeleton + case Volume => AnnotationLayerTypeProto.volume + } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala index a00e21e9a7c..e9fa3268163 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala @@ -7,7 +7,7 @@ trait AnnotationUpdateAction extends UpdateAction case class AddLayerAnnotationUpdateAction(layerName: String, tracingId: String, - typ: AnnotationLayerType, + `type`: AnnotationLayerType, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -21,7 +21,7 @@ case class AddLayerAnnotationUpdateAction(layerName: String, case class DeleteLayerAnnotationUpdateAction(tracingId: String, layerName: String, // Just stored for nicer-looking history - typ: AnnotationLayerType, // Just stored for nicer-looking history + `type`: AnnotationLayerType, // Just stored for nicer-looking history actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index ef5fe9ab81c..04ef116adc3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -2,14 +2,8 @@ package com.scalableminds.webknossos.tracingstore.annotation import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox -import com.scalableminds.webknossos.datastore.Annotation.{ - AddLayerAnnotationUpdateAction, - AnnotationLayerProto, - AnnotationProto, - DeleteLayerAnnotationUpdateAction, - UpdateLayerMetadataAnnotationUpdateAction, - UpdateMetadataAnnotationUpdateAction -} +import com.scalableminds.webknossos.datastore.Annotation.{AnnotationLayerProto, AnnotationProto} +import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{ CreateNodeSkeletonAction, DeleteNodeSkeletonAction, @@ -18,8 +12,8 @@ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{ import com.scalableminds.webknossos.tracingstore.tracings.volume.UpdateBucketVolumeAction import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingUpdatesReport} +import net.liftweb.common.{Empty, Full} import play.api.libs.json.{JsObject, JsValue, Json} -import scalapb.GeneratedMessage import javax.inject.Inject import scala.concurrent.ExecutionContext @@ -77,24 +71,26 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl } yield updateActionGroups.reverse.flatten } - def applyUpdate(annotation: AnnotationProto, updateAction: GeneratedMessage)( + def applyUpdate(annotation: AnnotationProto, updateAction: UpdateAction)( implicit ec: ExecutionContext): Fox[AnnotationProto] = for { - - withAppliedChange <- updateAction match { + updated <- updateAction match { case a: AddLayerAnnotationUpdateAction => Fox.successful( - annotation.copy(layers = annotation.layers :+ AnnotationLayerProto(a.tracingId, a.name, `type` = a.`type`))) + annotation.copy( + layers = annotation.layers :+ AnnotationLayerProto(a.tracingId, + a.layerName, + `type` = AnnotationLayerType.toProto(a.`type`)))) case a: DeleteLayerAnnotationUpdateAction => Fox.successful(annotation.copy(layers = annotation.layers.filter(_.tracingId != a.tracingId))) case a: UpdateLayerMetadataAnnotationUpdateAction => Fox.successful(annotation.copy(layers = annotation.layers.map(l => - if (l.tracingId == a.tracingId) l.copy(name = a.name) else l))) + if (l.tracingId == a.tracingId) l.copy(name = a.layerName) else l))) case a: UpdateMetadataAnnotationUpdateAction => Fox.successful(annotation.copy(name = a.name, description = a.description)) - case _ => Fox.failure("Received unsupported AnnotationUpdaetAction action") + case _ => Fox.failure("Received unsupported AnnotationUpdateAction action") } - } yield withAppliedChange.copy(version = withAppliedChange.version + 1L) + } yield updated.copy(version = updated.version + 1L) def updateActionLog(annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]): Fox[JsValue] = { def versionedTupleToJson(tuple: (Long, List[UpdateAction])): JsObject = @@ -112,6 +108,72 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl } yield Json.toJson(updateActionGroupsJs) } + def get(annotationId: String, version: Option[Long], applyUpdates: Boolean, userToken: Option[String])( + implicit ec: ExecutionContext): Fox[AnnotationProto] = + for { + annotationWithVersion <- tracingDataStore.annotations.get(annotationId, version)(fromProtoBytes[AnnotationProto]) + annotation = annotationWithVersion.value + updated <- if (applyUpdates) applyPendingUpdates(annotation, annotationId, version, userToken) + else Fox.successful(annotation) + } yield updated + + private def applyPendingUpdates(annotation: AnnotationProto, + annotationId: String, + targetVersionOpt: Option[Long], + userToken: Option[String])(implicit ec: ExecutionContext): Fox[AnnotationProto] = + for { + targetVersion <- determineTargetVersion(annotation, annotationId, targetVersionOpt) + updates <- findPendingUpdates(annotationId, annotation.version, targetVersion) + updated <- applyUpdates(annotation, annotationId, updates, targetVersion, userToken) + } yield updated + + private def applyUpdates(annotation: AnnotationProto, + annotationId: String, + updates: List[UpdateAction], + targetVersion: Long, + userToken: Option[String])(implicit ec: ExecutionContext): Fox[AnnotationProto] = { + + def updateIter(tracingFox: Fox[AnnotationProto], remainingUpdates: List[UpdateAction]): Fox[AnnotationProto] = + tracingFox.futureBox.flatMap { + case Empty => Fox.empty + case Full(annotation) => + remainingUpdates match { + case List() => Fox.successful(annotation) + case RevertToVersionUpdateAction(sourceVersion, _, _, _) :: tail => + val sourceTracing = get(annotationId, Some(sourceVersion), applyUpdates = true, userToken) + updateIter(sourceTracing, tail) + case update :: tail => updateIter(applyUpdate(annotation, update), tail) + } + case _ => tracingFox + } + + if (updates.isEmpty) Full(annotation) + else { + for { + updated <- updateIter(Some(annotation), updates) + } yield updated.withVersion(targetVersion) + } + } + + private def determineTargetVersion(annotation: AnnotationProto, + annotationId: String, + targetVersionOpt: Option[Long]): Fox[Long] = + /* + * Determines the newest saved version from the updates column. + * if there are no updates at all, assume annotation is brand new (possibly created from NML, + * hence the emptyFallbck annotation.version) + */ + for { + newestUpdateVersion <- tracingDataStore.annotationUpdates.getVersion(annotationId, + mayBeEmpty = Some(true), + emptyFallback = Some(annotation.version)) + } yield { + targetVersionOpt match { + case None => newestUpdateVersion + case Some(desiredSome) => math.min(desiredSome, newestUpdateVersion) + } + } + def updateActionStatistics(tracingId: String): Fox[JsObject] = for { updateActionGroups <- tracingDataStore.skeletonUpdates.getMultipleVersions(tracingId)( diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala index 04c58bf3235..ee0cfaf4820 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala @@ -86,9 +86,10 @@ object UpdateAction { deserialize[UpdateUserBoundingBoxVisibilitySkeletonAction](jsonValue) // Volume - case "updateBucket" => deserialize[UpdateBucketVolumeAction](jsonValue) - case "updateVolumeTracing" => deserialize[UpdateTracingVolumeAction](jsonValue) - case "updateUserBoundingBoxes" => deserialize[UpdateUserBoundingBoxesVolumeAction](jsonValue) + case "updateBucket" => deserialize[UpdateBucketVolumeAction](jsonValue) + case "updateVolumeTracing" => deserialize[UpdateTracingVolumeAction](jsonValue) + case "updateUserBoundingBoxes" => + deserialize[UpdateUserBoundingBoxesVolumeAction](jsonValue) // TODO: rename key (must be different from skeleton action) case "updateUserBoundingBoxVisibility" => deserialize[UpdateUserBoundingBoxVisibilityVolumeAction](jsonValue) case "removeFallbackLayer" => deserialize[RemoveFallbackLayerVolumeAction](jsonValue) case "importVolumeTracing" => deserialize[ImportVolumeDataVolumeAction](jsonValue) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index cf65236e3b5..6739b3697d0 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -77,6 +77,23 @@ class TSAnnotationController @Inject()( } } + def get(token: Option[String], annotationId: String, version: Option[Long]): Action[AnyContent] = + Action.async { implicit request => + log() { + logTime(slackNotificationService.noticeSlowRequest) { + accessTokenService.validateAccess(UserAccessRequest.readAnnotation(annotationId), + urlOrHeaderToken(token, request)) { + for { + annotationProto <- annotationService.get(annotationId, + version, + applyUpdates = false, + urlOrHeaderToken(token, request)) + } yield Ok(annotationProto.toByteArray).as(protobufMimeType) + } + } + } + } + } // get version history diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala index 4de87d378b3..f6651c5d090 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala @@ -22,8 +22,6 @@ class TracingDataStore @Inject()(config: TracingStoreConfig, lazy val skeletons = new FossilDBClient("skeletons", config, slackNotificationService) - lazy val annotationUpdates = new FossilDBClient("annotationUpdates", config, slackNotificationService) - lazy val skeletonUpdates = new FossilDBClient("skeletonUpdates", config, slackNotificationService) lazy val volumes = new FossilDBClient("volumes", config, slackNotificationService) @@ -46,6 +44,8 @@ class TracingDataStore @Inject()(config: TracingStoreConfig, lazy val annotations = new FossilDBClient("annotations", config, slackNotificationService) + lazy val annotationUpdates = new FossilDBClient("annotationUpdates", config, slackNotificationService) + private def shutdown(): Unit = { healthClient.shutdown() skeletons.shutdown() diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index d4079a604d8..c641445e98c 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -6,7 +6,8 @@ GET /health @com.scalableminds.webknossos.tracingstore.controllers.Application.health POST /annotation/initialize @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.initialize(token: Option[String], annotationId: String) -POST /annotation/update @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.update(token: Option[String], annotationId: String) +GET /annotation/:annotationId @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.get(token: Option[String], annotationId: String, version: Option[Long]) +POST /annotation/:annotationId/update @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.update(token: Option[String], annotationId: String) POST /annotation/:annotationId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionLog(token: Option[String], annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]) GET /annotation/:annotationId/updateActionStatistics @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionStatistics(token: Option[String], annotationId: String) From e5f128edeaa032a2b09f1ef42807d9cb0ae092ea Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 29 Jul 2024 11:36:32 +0200 Subject: [PATCH 025/150] TracingCollection for updating --- .../annotation/TSAnnotationService.scala | 67 ++++++++++++++----- 1 file changed, 52 insertions(+), 15 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 04ef116adc3..8dbebe7ff9f 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -3,13 +3,16 @@ package com.scalableminds.webknossos.tracingstore.annotation import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.Annotation.{AnnotationLayerProto, AnnotationProto} +import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing +import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{ CreateNodeSkeletonAction, DeleteNodeSkeletonAction, + SkeletonUpdateAction, UpdateTracingSkeletonAction } -import com.scalableminds.webknossos.tracingstore.tracings.volume.UpdateBucketVolumeAction +import com.scalableminds.webknossos.tracingstore.tracings.volume.{UpdateBucketVolumeAction, VolumeUpdateAction} import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingUpdatesReport} import net.liftweb.common.{Empty, Full} @@ -18,6 +21,8 @@ import play.api.libs.json.{JsObject, JsValue, Json} import javax.inject.Inject import scala.concurrent.ExecutionContext +case class TracingCollection(tracingsById: Map[String, Either[SkeletonTracing, VolumeTracing]]) {} + class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosClient, tracingDataStore: TracingDataStore) extends KeyValueStoreImplicits { @@ -71,8 +76,9 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl } yield updateActionGroups.reverse.flatten } - def applyUpdate(annotation: AnnotationProto, updateAction: UpdateAction)( - implicit ec: ExecutionContext): Fox[AnnotationProto] = + private def applyUpdate(annotationWithTracings: (AnnotationProto, TracingCollection), updateAction: UpdateAction)( + implicit ec: ExecutionContext): Fox[(AnnotationProto, TracingCollection)] = { + val annotation = annotationWithTracings._1 for { updated <- updateAction match { case a: AddLayerAnnotationUpdateAction => @@ -90,7 +96,8 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl Fox.successful(annotation.copy(name = a.name, description = a.description)) case _ => Fox.failure("Received unsupported AnnotationUpdateAction action") } - } yield updated.copy(version = updated.version + 1L) + } yield (updated.copy(version = updated.version + 1L), annotationWithTracings._2) + } def updateActionLog(annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]): Fox[JsValue] = { def versionedTupleToJson(tuple: (Long, List[UpdateAction])): JsObject = @@ -124,34 +131,64 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl for { targetVersion <- determineTargetVersion(annotation, annotationId, targetVersionOpt) updates <- findPendingUpdates(annotationId, annotation.version, targetVersion) - updated <- applyUpdates(annotation, annotationId, updates, targetVersion, userToken) + tracingCollection <- findTracingsForUpdates(annotation, updates) + updated <- applyUpdates(annotation, tracingCollection, annotationId, updates, targetVersion, userToken) } yield updated + private def findTracingsForUpdates(annotation: AnnotationProto, updates: List[UpdateAction])( + implicit ec: ExecutionContext): Fox[TracingCollection] = { + val skeletonTracingIds = updates.flatMap { + case u: SkeletonUpdateAction => Some(u.actionTracingId) + case _ => None + } + val volumeTracingIds = updates.flatMap { + case u: VolumeUpdateAction => Some(u.actionTracingId) + case _ => None + } + for { + skeletonTracings <- Fox.serialCombined(skeletonTracingIds)( + id => + tracingDataStore.skeletons.get[SkeletonTracing](id, Some(annotation.version), mayBeEmpty = Some(true))( + fromProtoBytes[SkeletonTracing])) + volumeTracings <- Fox.serialCombined(volumeTracingIds)( + id => + tracingDataStore.volumes + .get[VolumeTracing](id, Some(annotation.version), mayBeEmpty = Some(true))(fromProtoBytes[VolumeTracing])) + skeletonTracingsMap: Map[String, Either[SkeletonTracing, VolumeTracing]] = skeletonTracingIds + .zip(skeletonTracings.map(versioned => Left[SkeletonTracing, VolumeTracing](versioned.value))) + .toMap + volumeTracingsMap: Map[String, Either[SkeletonTracing, VolumeTracing]] = volumeTracingIds + .zip(volumeTracings.map(versioned => Right[SkeletonTracing, VolumeTracing](versioned.value))) + .toMap + } yield TracingCollection(skeletonTracingsMap ++ volumeTracingsMap) + } + private def applyUpdates(annotation: AnnotationProto, + tracingCollection: TracingCollection, annotationId: String, updates: List[UpdateAction], targetVersion: Long, userToken: Option[String])(implicit ec: ExecutionContext): Fox[AnnotationProto] = { - def updateIter(tracingFox: Fox[AnnotationProto], remainingUpdates: List[UpdateAction]): Fox[AnnotationProto] = - tracingFox.futureBox.flatMap { + def updateIter(annotationWithTracingsFox: Fox[(AnnotationProto, TracingCollection)], + remainingUpdates: List[UpdateAction]): Fox[(AnnotationProto, TracingCollection)] = + annotationWithTracingsFox.futureBox.flatMap { case Empty => Fox.empty - case Full(annotation) => + case Full(annotationWithTracings) => remainingUpdates match { - case List() => Fox.successful(annotation) + case List() => Fox.successful(annotationWithTracings) case RevertToVersionUpdateAction(sourceVersion, _, _, _) :: tail => - val sourceTracing = get(annotationId, Some(sourceVersion), applyUpdates = true, userToken) - updateIter(sourceTracing, tail) - case update :: tail => updateIter(applyUpdate(annotation, update), tail) + ??? + case update :: tail => updateIter(applyUpdate(annotationWithTracings, update), tail) } - case _ => tracingFox + case _ => annotationWithTracingsFox } if (updates.isEmpty) Full(annotation) else { for { - updated <- updateIter(Some(annotation), updates) - } yield updated.withVersion(targetVersion) + updated <- updateIter(Some((annotation, tracingCollection)), updates) + } yield updated._1.withVersion(targetVersion) } } From 201a1c90608512052544f1e9017c633c13fad83c Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 29 Jul 2024 12:02:59 +0200 Subject: [PATCH 026/150] AnnotationWithTracings --- .../annotation/TSAnnotationService.scala | 76 ++++++++++++------- 1 file changed, 50 insertions(+), 26 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 8dbebe7ff9f..9d83a840e4b 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -21,6 +21,38 @@ import play.api.libs.json.{JsObject, JsValue, Json} import javax.inject.Inject import scala.concurrent.ExecutionContext +case class AnnotationWithTracings(annotation: AnnotationProto, + tracingsById: Map[String, Either[SkeletonTracing, VolumeTracing]]) { + def getSkeleton(tracingId: String): SkeletonTracing = ??? + + def version: Long = annotation.version + + def addTracing(a: AddLayerAnnotationUpdateAction): AnnotationWithTracings = + AnnotationWithTracings( + annotation.copy( + layers = annotation.layers :+ AnnotationLayerProto(a.tracingId, + a.layerName, + `type` = AnnotationLayerType.toProto(a.`type`))), + tracingsById) + + def deleteTracing(a: DeleteLayerAnnotationUpdateAction): AnnotationWithTracings = + AnnotationWithTracings(annotation.copy(layers = annotation.layers.filter(_.tracingId != a.tracingId)), tracingsById) + + def updateLayerMetadata(a: UpdateLayerMetadataAnnotationUpdateAction): AnnotationWithTracings = + AnnotationWithTracings(annotation.copy(layers = annotation.layers.map(l => + if (l.tracingId == a.tracingId) l.copy(name = a.layerName) else l)), + tracingsById) + + def updateMetadata(a: UpdateMetadataAnnotationUpdateAction): AnnotationWithTracings = + AnnotationWithTracings(annotation.copy(name = a.name, description = a.description), tracingsById) + + def incrementVersion: AnnotationWithTracings = + AnnotationWithTracings(annotation.copy(version = annotation.version + 1L), tracingsById) + + def withVersion(newVersion: Long): AnnotationWithTracings = + AnnotationWithTracings(annotation.copy(version = newVersion), tracingsById) +} + case class TracingCollection(tracingsById: Map[String, Either[SkeletonTracing, VolumeTracing]]) {} class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosClient, @@ -76,28 +108,21 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl } yield updateActionGroups.reverse.flatten } - private def applyUpdate(annotationWithTracings: (AnnotationProto, TracingCollection), updateAction: UpdateAction)( - implicit ec: ExecutionContext): Fox[(AnnotationProto, TracingCollection)] = { - val annotation = annotationWithTracings._1 + private def applyUpdate(annotationWithTracings: AnnotationWithTracings, updateAction: UpdateAction)( + implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { updated <- updateAction match { case a: AddLayerAnnotationUpdateAction => - Fox.successful( - annotation.copy( - layers = annotation.layers :+ AnnotationLayerProto(a.tracingId, - a.layerName, - `type` = AnnotationLayerType.toProto(a.`type`)))) + Fox.successful(annotationWithTracings.addTracing(a)) case a: DeleteLayerAnnotationUpdateAction => - Fox.successful(annotation.copy(layers = annotation.layers.filter(_.tracingId != a.tracingId))) + Fox.successful(annotationWithTracings.deleteTracing(a)) case a: UpdateLayerMetadataAnnotationUpdateAction => - Fox.successful(annotation.copy(layers = annotation.layers.map(l => - if (l.tracingId == a.tracingId) l.copy(name = a.layerName) else l))) + Fox.successful(annotationWithTracings.updateLayerMetadata(a)) case a: UpdateMetadataAnnotationUpdateAction => - Fox.successful(annotation.copy(name = a.name, description = a.description)) + Fox.successful(annotationWithTracings.updateMetadata(a)) case _ => Fox.failure("Received unsupported AnnotationUpdateAction action") } - } yield (updated.copy(version = updated.version + 1L), annotationWithTracings._2) - } + } yield updated.incrementVersion def updateActionLog(annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]): Fox[JsValue] = { def versionedTupleToJson(tuple: (Long, List[UpdateAction])): JsObject = @@ -131,12 +156,12 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl for { targetVersion <- determineTargetVersion(annotation, annotationId, targetVersionOpt) updates <- findPendingUpdates(annotationId, annotation.version, targetVersion) - tracingCollection <- findTracingsForUpdates(annotation, updates) - updated <- applyUpdates(annotation, tracingCollection, annotationId, updates, targetVersion, userToken) - } yield updated + annotationWithTracings <- findTracingsForUpdates(annotation, updates) + updated <- applyUpdates(annotationWithTracings, annotationId, updates, targetVersion, userToken) + } yield updated.annotation private def findTracingsForUpdates(annotation: AnnotationProto, updates: List[UpdateAction])( - implicit ec: ExecutionContext): Fox[TracingCollection] = { + implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = { val skeletonTracingIds = updates.flatMap { case u: SkeletonUpdateAction => Some(u.actionTracingId) case _ => None @@ -160,18 +185,17 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl volumeTracingsMap: Map[String, Either[SkeletonTracing, VolumeTracing]] = volumeTracingIds .zip(volumeTracings.map(versioned => Right[SkeletonTracing, VolumeTracing](versioned.value))) .toMap - } yield TracingCollection(skeletonTracingsMap ++ volumeTracingsMap) + } yield AnnotationWithTracings(annotation, skeletonTracingsMap ++ volumeTracingsMap) } - private def applyUpdates(annotation: AnnotationProto, - tracingCollection: TracingCollection, + private def applyUpdates(annotation: AnnotationWithTracings, annotationId: String, updates: List[UpdateAction], targetVersion: Long, - userToken: Option[String])(implicit ec: ExecutionContext): Fox[AnnotationProto] = { + userToken: Option[String])(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = { - def updateIter(annotationWithTracingsFox: Fox[(AnnotationProto, TracingCollection)], - remainingUpdates: List[UpdateAction]): Fox[(AnnotationProto, TracingCollection)] = + def updateIter(annotationWithTracingsFox: Fox[AnnotationWithTracings], + remainingUpdates: List[UpdateAction]): Fox[AnnotationWithTracings] = annotationWithTracingsFox.futureBox.flatMap { case Empty => Fox.empty case Full(annotationWithTracings) => @@ -187,8 +211,8 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl if (updates.isEmpty) Full(annotation) else { for { - updated <- updateIter(Some((annotation, tracingCollection)), updates) - } yield updated._1.withVersion(targetVersion) + updated <- updateIter(Some(annotation), updates) + } yield updated.withVersion(targetVersion) } } From fd5ccd33721a0bfe66e88d08e0e9ac4f79f48f82 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 29 Jul 2024 13:31:54 +0200 Subject: [PATCH 027/150] applyOn skeleton --- .../annotation/TSAnnotationService.scala | 16 +++- .../updating/SkeletonUpdateActions.scala | 78 +++++++++---------- 2 files changed, 53 insertions(+), 41 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 9d83a840e4b..c9a818ea636 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -2,6 +2,7 @@ package com.scalableminds.webknossos.tracingstore.annotation import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox +import com.scalableminds.util.tools.Fox.option2Fox import com.scalableminds.webknossos.datastore.Annotation.{AnnotationLayerProto, AnnotationProto} import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing @@ -51,9 +52,17 @@ case class AnnotationWithTracings(annotation: AnnotationProto, def withVersion(newVersion: Long): AnnotationWithTracings = AnnotationWithTracings(annotation.copy(version = newVersion), tracingsById) -} -case class TracingCollection(tracingsById: Map[String, Either[SkeletonTracing, VolumeTracing]]) {} + def applySkeletonAction(a: SkeletonUpdateAction)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = + for { + skeletonTracingEither <- tracingsById.get(a.actionTracingId).toFox + skeletonTracing <- skeletonTracingEither match { + case Left(st: SkeletonTracing) => Fox.successful(st) + case _ => Fox.failure("wrong tracing type") + } + updated = a.applyOn(skeletonTracing) + } yield AnnotationWithTracings(annotation, tracingsById.updated(a.actionTracingId, Left(updated))) +} class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosClient, tracingDataStore: TracingDataStore) @@ -113,6 +122,7 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl for { updated <- updateAction match { case a: AddLayerAnnotationUpdateAction => + // TODO create tracing object (ask wk for needed parameters e.g. fallback layer info?) Fox.successful(annotationWithTracings.addTracing(a)) case a: DeleteLayerAnnotationUpdateAction => Fox.successful(annotationWithTracings.deleteTracing(a)) @@ -120,6 +130,8 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl Fox.successful(annotationWithTracings.updateLayerMetadata(a)) case a: UpdateMetadataAnnotationUpdateAction => Fox.successful(annotationWithTracings.updateMetadata(a)) + case a: SkeletonUpdateAction => + annotationWithTracings.applySkeletonAction(a) case _ => Fox.failure("Received unsupported AnnotationUpdateAction action") } } yield updated.incrementVersion diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala index cf43dc92e47..0fe3c367958 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala @@ -2,13 +2,16 @@ package com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.util.geometry.{Vec3Double, Vec3Int} -import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits +import com.scalableminds.webknossos.datastore.SkeletonTracing.{Edge, Node, SkeletonTracing, Tree, TreeGroup} +import com.scalableminds.webknossos.datastore.helpers.{NodeDefaults, ProtoGeometryImplicits} import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate import com.scalableminds.webknossos.tracingstore.annotation.{LayerUpdateAction, UpdateAction} import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.TreeType.TreeType import play.api.libs.json._ -trait SkeletonUpdateAction extends LayerUpdateAction +trait SkeletonUpdateAction extends LayerUpdateAction { + def applyOn(tracing: SkeletonTracing): SkeletonTracing +} case class CreateTreeSkeletonAction(id: Int, color: Option[com.scalableminds.util.image.Color], @@ -26,7 +29,7 @@ case class CreateTreeSkeletonAction(id: Int, info: Option[String] = None) extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { val newTree = Tree( id, Nil, @@ -42,7 +45,7 @@ case class CreateTreeSkeletonAction(id: Int, edgesAreVisible ) tracing.withTrees(newTree +: tracing.trees) - }*/ + } override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -57,8 +60,8 @@ case class DeleteTreeSkeletonAction(id: Int, actionAuthorId: Option[String] = None, info: Option[String] = None) extends SkeletonUpdateAction { - /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = - tracing.withTrees(tracing.trees.filter(_.treeId != id))*/ + override def applyOn(tracing: SkeletonTracing): SkeletonTracing = + tracing.withTrees(tracing.trees.filter(_.treeId != id)) override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -83,7 +86,7 @@ case class UpdateTreeSkeletonAction(id: Int, info: Option[String] = None) extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { def treeTransform(tree: Tree) = tree.copy( color = colorOptToProto(color).orElse(tree.color), @@ -96,7 +99,7 @@ case class UpdateTreeSkeletonAction(id: Int, ) tracing.withTrees(mapTrees(tracing, id, treeTransform)) - }*/ + } override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -113,7 +116,7 @@ case class MergeTreeSkeletonAction(sourceId: Int, info: Option[String] = None) extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - /* + // only nodes and edges are merged here, // other properties are managed explicitly // by the frontend with extra actions @@ -126,7 +129,7 @@ case class MergeTreeSkeletonAction(sourceId: Int, } tracing.withTrees(mapTrees(tracing, targetId, treeTransform).filter(_.treeId != sourceId)) - }*/ + } override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -147,7 +150,6 @@ case class MoveTreeComponentSkeletonAction(nodeIds: List[Int], extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - /* // this should only move a whole component, // that is disjoint from the rest of the tree override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { @@ -170,7 +172,6 @@ case class MoveTreeComponentSkeletonAction(nodeIds: List[Int], tracing.withTrees(tracing.trees.map(selectTree)) } - */ override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -190,10 +191,10 @@ case class CreateEdgeSkeletonAction(source: Int, info: Option[String] = None) extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { def treeTransform(tree: Tree) = tree.withEdges(Edge(source, target) +: tree.edges) tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) - }*/ + } override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -213,10 +214,10 @@ case class DeleteEdgeSkeletonAction(source: Int, info: Option[String] = None) extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { def treeTransform(tree: Tree) = tree.copy(edges = tree.edges.filter(_ != Edge(source, target))) tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) - }*/ + } override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -245,7 +246,7 @@ case class CreateNodeSkeletonAction(id: Int, extends SkeletonUpdateAction with SkeletonUpdateActionHelper with ProtoGeometryImplicits { - /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { val rotationOrDefault = rotation getOrElse NodeDefaults.rotation val newNode = Node( id, @@ -263,7 +264,7 @@ case class CreateNodeSkeletonAction(id: Int, def treeTransform(tree: Tree) = tree.withNodes(newNode +: tree.nodes) tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) - }*/ + } override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -292,7 +293,7 @@ case class UpdateNodeSkeletonAction(id: Int, extends SkeletonUpdateAction with SkeletonUpdateActionHelper with ProtoGeometryImplicits { - /* override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { val rotationOrDefault = rotation getOrElse NodeDefaults.rotation val newNode = Node( @@ -313,7 +314,6 @@ case class UpdateNodeSkeletonAction(id: Int, tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) } - */ override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -333,13 +333,13 @@ case class DeleteNodeSkeletonAction(nodeId: Int, info: Option[String] = None) extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { def treeTransform(tree: Tree) = tree.withNodes(tree.nodes.filter(_.id != nodeId)) tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) - }*/ + } override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -357,8 +357,8 @@ case class UpdateTreeGroupsSkeletonAction(treeGroups: List[UpdateActionTreeGroup info: Option[String] = None) extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = - tracing.withTreeGroups(treeGroups.map(convertTreeGroup))*/ + override def applyOn(tracing: SkeletonTracing): SkeletonTracing = + tracing.withTreeGroups(treeGroups.map(convertTreeGroup)) override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -381,7 +381,7 @@ case class UpdateTracingSkeletonAction(activeNode: Option[Int], editPositionAdditionalCoordinates: Option[Seq[AdditionalCoordinate]] = None) extends SkeletonUpdateAction with ProtoGeometryImplicits { - /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = + override def applyOn(tracing: SkeletonTracing): SkeletonTracing = tracing.copy( editPosition = editPosition, editRotation = editRotation, @@ -389,7 +389,7 @@ case class UpdateTracingSkeletonAction(activeNode: Option[Int], userBoundingBox = userBoundingBox, activeNodeId = activeNode, editPositionAdditionalCoordinates = AdditionalCoordinate.toProto(editPositionAdditionalCoordinates) - )*/ + ) override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -406,8 +406,8 @@ case class RevertToVersionSkeletonAction(sourceVersion: Long, actionAuthorId: Option[String] = None, info: Option[String] = None) extends SkeletonUpdateAction { - /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = - throw new Exception("RevertToVersionAction applied on unversioned tracing")*/ + override def applyOn(tracing: SkeletonTracing): SkeletonTracing = + throw new Exception("RevertToVersionAction applied on unversioned tracing") override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -426,11 +426,11 @@ case class UpdateTreeVisibilitySkeletonAction(treeId: Int, info: Option[String] = None) extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { def treeTransform(tree: Tree) = tree.copy(isVisible = Some(isVisible)) tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) - }*/ + } override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -449,7 +449,7 @@ case class UpdateTreeGroupVisibilitySkeletonAction(treeGroupId: Option[Int], info: Option[String] = None) extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { def updateTreeGroups(treeGroups: Seq[TreeGroup]) = { def treeTransform(tree: Tree) = if (treeGroups.exists(group => tree.groupId.contains(group.groupId))) @@ -469,7 +469,7 @@ case class UpdateTreeGroupVisibilitySkeletonAction(treeGroupId: Option[Int], .map(group => updateTreeGroups(GroupUtils.getAllChildrenTreeGroups(group))) .getOrElse(tracing) } - }*/ + } override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -489,11 +489,11 @@ case class UpdateTreeEdgesVisibilitySkeletonAction(treeId: Int, extends SkeletonUpdateAction with SkeletonUpdateActionHelper { - /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { def treeTransform(tree: Tree) = tree.copy(edgesAreVisible = Some(edgesAreVisible)) tracing.withTrees(mapTrees(tracing, treeId, treeTransform)) - }*/ + } override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -510,8 +510,8 @@ case class UpdateUserBoundingBoxesSkeletonAction(boundingBoxes: List[NamedBoundi actionAuthorId: Option[String] = None, info: Option[String] = None) extends SkeletonUpdateAction { - /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = - tracing.withUserBoundingBoxes(boundingBoxes.map(_.toProto))*/ + override def applyOn(tracing: SkeletonTracing): SkeletonTracing = + tracing.withUserBoundingBoxes(boundingBoxes.map(_.toProto)) override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -529,7 +529,7 @@ case class UpdateUserBoundingBoxVisibilitySkeletonAction(boundingBoxId: Option[I actionAuthorId: Option[String] = None, info: Option[String] = None) extends SkeletonUpdateAction { - /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { + override def applyOn(tracing: SkeletonTracing): SkeletonTracing = { def updateUserBoundingBoxes() = tracing.userBoundingBoxes.map { boundingBox => if (boundingBoxId.forall(_ == boundingBox.id)) @@ -539,7 +539,7 @@ case class UpdateUserBoundingBoxVisibilitySkeletonAction(boundingBoxId: Option[I } tracing.withUserBoundingBoxes(updateUserBoundingBoxes()) - }*/ + } override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -556,7 +556,7 @@ case class UpdateTdCameraSkeletonAction(actionTimestamp: Option[Long] = None, info: Option[String] = None) extends SkeletonUpdateAction { - /*override def applyOn(tracing: SkeletonTracing): SkeletonTracing = tracing*/ + override def applyOn(tracing: SkeletonTracing): SkeletonTracing = tracing override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) From d8a6ddb40f6c2468cc491eef87ca7cbdeb8effb0 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 29 Jul 2024 13:42:50 +0200 Subject: [PATCH 028/150] ApplyableVolumeUpdateAction --- .../annotation/TSAnnotationService.scala | 24 +++++++- .../volume/VolumeTracingService.scala | 4 +- .../tracings/volume/VolumeUpdateActions.scala | 61 ++++++++++--------- 3 files changed, 55 insertions(+), 34 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index c9a818ea636..8f1d05cbfcf 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -13,7 +13,11 @@ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{ SkeletonUpdateAction, UpdateTracingSkeletonAction } -import com.scalableminds.webknossos.tracingstore.tracings.volume.{UpdateBucketVolumeAction, VolumeUpdateAction} +import com.scalableminds.webknossos.tracingstore.tracings.volume.{ + ApplyableVolumeUpdateAction, + UpdateBucketVolumeAction, + VolumeUpdateAction +} import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingUpdatesReport} import net.liftweb.common.{Empty, Full} @@ -51,7 +55,7 @@ case class AnnotationWithTracings(annotation: AnnotationProto, AnnotationWithTracings(annotation.copy(version = annotation.version + 1L), tracingsById) def withVersion(newVersion: Long): AnnotationWithTracings = - AnnotationWithTracings(annotation.copy(version = newVersion), tracingsById) + AnnotationWithTracings(annotation.copy(version = newVersion), tracingsById) // TODO also update version in tracings? def applySkeletonAction(a: SkeletonUpdateAction)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { @@ -62,6 +66,16 @@ case class AnnotationWithTracings(annotation: AnnotationProto, } updated = a.applyOn(skeletonTracing) } yield AnnotationWithTracings(annotation, tracingsById.updated(a.actionTracingId, Left(updated))) + + def applyVolumeAction(a: ApplyableVolumeUpdateAction)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = + for { + volumeTracingEither <- tracingsById.get(a.actionTracingId).toFox + volumeTracing <- volumeTracingEither match { + case Right(vt: VolumeTracing) => Fox.successful(vt) + case _ => Fox.failure("wrong tracing type") + } + updated = a.applyOn(volumeTracing) + } yield AnnotationWithTracings(annotation, tracingsById.updated(a.actionTracingId, Right(updated))) } class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosClient, @@ -132,9 +146,13 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl Fox.successful(annotationWithTracings.updateMetadata(a)) case a: SkeletonUpdateAction => annotationWithTracings.applySkeletonAction(a) + case a: ApplyableVolumeUpdateAction => + annotationWithTracings.applyVolumeAction(a) + case a: VolumeUpdateAction => + Fox.successful(annotationWithTracings) // TODO case _ => Fox.failure("Received unsupported AnnotationUpdateAction action") } - } yield updated.incrementVersion + } yield updated def updateActionLog(annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]): Fox[JsValue] = { def versionedTupleToJson(tuple: (Long, List[UpdateAction])): JsObject = diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 53d507dbfd9..792a893d3d6 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -114,7 +114,7 @@ class VolumeTracingService @Inject()( mappingName, editableMappingTracingId) ?~> "volumeSegmentIndex.update.failed" - private def applyUpdateOn(tracing: VolumeTracing, update: ApplyableVolumeAction): VolumeTracing = ??? + private def applyUpdateOn(tracing: VolumeTracing, update: ApplyableVolumeUpdateAction): VolumeTracing = ??? def handleUpdateGroup(tracingId: String, updateGroup: UpdateActionGroup, @@ -163,7 +163,7 @@ class VolumeTracingService @Inject()( } else deleteSegmentData(tracingId, tracing, a, segmentIndexBuffer, updateGroup.version, userToken) ?~> "Failed to delete segment data." case _: UpdateTdCameraVolumeAction => Fox.successful(tracing) - case a: ApplyableVolumeAction => Fox.successful(applyUpdateOn(tracing, a)) + case a: ApplyableVolumeUpdateAction => Fox.successful(applyUpdateOn(tracing, a)) case _ => Fox.failure("Unknown action.") } case Empty => diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala index 625811c2f0e..a4b3ef91ac5 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala @@ -3,6 +3,7 @@ package com.scalableminds.webknossos.tracingstore.tracings.volume import java.util.Base64 import com.scalableminds.util.geometry.{Vec3Double, Vec3Int} import com.scalableminds.webknossos.datastore.VolumeTracing.{Segment, SegmentGroup, VolumeTracing} +import com.scalableminds.webknossos.datastore.geometry.NamedBoundingBoxProto import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate import com.scalableminds.webknossos.tracingstore.annotation.{LayerUpdateAction, UpdateAction} @@ -27,7 +28,9 @@ trait VolumeUpdateActionHelper { trait VolumeUpdateAction extends LayerUpdateAction -trait ApplyableVolumeAction extends VolumeUpdateAction +trait ApplyableVolumeUpdateAction extends VolumeUpdateAction { + def applyOn(tracing: VolumeTracing): VolumeTracing +} case class UpdateBucketVolumeAction(position: Vec3Int, cubeSize: Int, @@ -88,15 +91,15 @@ case class UpdateUserBoundingBoxesVolumeAction(boundingBoxes: List[NamedBounding actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends ApplyableVolumeAction { + extends ApplyableVolumeUpdateAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - /*override def applyOn(tracing: VolumeTracing): VolumeTracing = - tracing.withUserBoundingBoxes(boundingBoxes.map(_.toProto))*/ + override def applyOn(tracing: VolumeTracing): VolumeTracing = + tracing.withUserBoundingBoxes(boundingBoxes.map(_.toProto)) } case class UpdateUserBoundingBoxVisibilityVolumeAction(boundingBoxId: Option[Int], @@ -105,7 +108,7 @@ case class UpdateUserBoundingBoxVisibilityVolumeAction(boundingBoxId: Option[Int actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends ApplyableVolumeAction { + extends ApplyableVolumeUpdateAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) @@ -113,9 +116,9 @@ case class UpdateUserBoundingBoxVisibilityVolumeAction(boundingBoxId: Option[Int override def isViewOnlyChange: Boolean = true - /*override def applyOn(tracing: VolumeTracing): VolumeTracing = { + override def applyOn(tracing: VolumeTracing): VolumeTracing = { - def updateUserBoundingBoxes(): Seq[geometry.NamedBoundingBoxProto] = + def updateUserBoundingBoxes(): Seq[NamedBoundingBoxProto] = tracing.userBoundingBoxes.map { boundingBox => if (boundingBoxId.forall(_ == boundingBox.id)) boundingBox.copy(isVisible = Some(isVisible)) @@ -124,21 +127,21 @@ case class UpdateUserBoundingBoxVisibilityVolumeAction(boundingBoxId: Option[Int } tracing.withUserBoundingBoxes(updateUserBoundingBoxes()) - }*/ + } } case class RemoveFallbackLayerVolumeAction(actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends ApplyableVolumeAction { + extends ApplyableVolumeUpdateAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - /*override def applyOn(tracing: VolumeTracing): VolumeTracing = - tracing.clearFallbackLayer*/ + override def applyOn(tracing: VolumeTracing): VolumeTracing = + tracing.clearFallbackLayer } case class ImportVolumeDataVolumeAction(actionTracingId: String, @@ -146,28 +149,28 @@ case class ImportVolumeDataVolumeAction(actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends ApplyableVolumeAction { + extends ApplyableVolumeUpdateAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - /*override def applyOn(tracing: VolumeTracing): VolumeTracing = - tracing.copy(largestSegmentId = largestSegmentId)*/ + override def applyOn(tracing: VolumeTracing): VolumeTracing = + tracing.copy(largestSegmentId = largestSegmentId) } case class AddSegmentIndexVolumeAction(actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends ApplyableVolumeAction { + extends ApplyableVolumeUpdateAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - /*override def applyOn(tracing: VolumeTracing): VolumeTracing = - tracing.copy(hasSegmentIndex = Some(true))*/ + override def applyOn(tracing: VolumeTracing): VolumeTracing = + tracing.copy(hasSegmentIndex = Some(true)) } @@ -197,7 +200,7 @@ case class CreateSegmentVolumeAction(id: Long, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends ApplyableVolumeAction + extends ApplyableVolumeUpdateAction with ProtoGeometryImplicits { override def addTimestamp(timestamp: Long): VolumeUpdateAction = @@ -206,7 +209,7 @@ case class CreateSegmentVolumeAction(id: Long, this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - /*override def applyOn(tracing: VolumeTracing): VolumeTracing = { + override def applyOn(tracing: VolumeTracing): VolumeTracing = { val newSegment = Segment(id, anchorPosition.map(vec3IntToProto), @@ -216,7 +219,7 @@ case class CreateSegmentVolumeAction(id: Long, groupId, AdditionalCoordinate.toProto(additionalCoordinates)) tracing.addSegments(newSegment) - }*/ + } } case class UpdateSegmentVolumeAction(id: Long, @@ -230,7 +233,7 @@ case class UpdateSegmentVolumeAction(id: Long, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends ApplyableVolumeAction + extends ApplyableVolumeUpdateAction with ProtoGeometryImplicits with VolumeUpdateActionHelper { @@ -240,7 +243,7 @@ case class UpdateSegmentVolumeAction(id: Long, this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - /*override def applyOn(tracing: VolumeTracing): VolumeTracing = { + override def applyOn(tracing: VolumeTracing): VolumeTracing = { def segmentTransform(segment: Segment): Segment = segment.copy( anchorPosition = anchorPosition.map(vec3IntToProto), @@ -251,7 +254,7 @@ case class UpdateSegmentVolumeAction(id: Long, anchorPositionAdditionalCoordinates = AdditionalCoordinate.toProto(additionalCoordinates) ) tracing.withSegments(mapSegments(tracing, id, segmentTransform)) - }*/ + } } case class DeleteSegmentVolumeAction(id: Long, @@ -291,18 +294,18 @@ case class UpdateMappingNameVolumeAction(mappingName: Option[String], actionTimestamp: Option[Long], actionAuthorId: Option[String] = None, info: Option[String] = None) - extends ApplyableVolumeAction { + extends ApplyableVolumeUpdateAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - /* override def applyOn(tracing: VolumeTracing): VolumeTracing = + override def applyOn(tracing: VolumeTracing): VolumeTracing = if (tracing.mappingIsLocked.getOrElse(false)) tracing // cannot change mapping name if it is locked else tracing.copy(mappingName = mappingName, mappingIsEditable = Some(isEditable.getOrElse(false)), - mappingIsLocked = Some(isLocked.getOrElse(false)))*/ + mappingIsLocked = Some(isLocked.getOrElse(false))) } case class UpdateSegmentGroupsVolumeAction(segmentGroups: List[UpdateActionSegmentGroup], @@ -310,10 +313,10 @@ case class UpdateSegmentGroupsVolumeAction(segmentGroups: List[UpdateActionSegme actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends VolumeUpdateAction + extends ApplyableVolumeUpdateAction with VolumeUpdateActionHelper { - /*override def applyOn(tracing: VolumeTracing): VolumeTracing = - tracing.withSegmentGroups(segmentGroups.map(convertSegmentGroup))*/ + override def applyOn(tracing: VolumeTracing): VolumeTracing = + tracing.withSegmentGroups(segmentGroups.map(convertSegmentGroup)) override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = From 54444aab3d77bca67c31fc1d863e5be1b215a4bd Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 30 Jul 2024 11:59:45 +0200 Subject: [PATCH 029/150] make more update actions applyable --- .../annotation/TSAnnotationService.scala | 33 ++++++++++------ .../volume/VolumeTracingService.scala | 19 ++------- .../tracings/volume/VolumeUpdateActions.scala | 39 +++++++++---------- 3 files changed, 43 insertions(+), 48 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 8f1d05cbfcf..626d6519141 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -20,7 +20,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ } import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingUpdatesReport} -import net.liftweb.common.{Empty, Full} +import net.liftweb.common.{Box, Empty, Failure, Full} import play.api.libs.json.{JsObject, JsValue, Json} import javax.inject.Inject @@ -28,7 +28,24 @@ import scala.concurrent.ExecutionContext case class AnnotationWithTracings(annotation: AnnotationProto, tracingsById: Map[String, Either[SkeletonTracing, VolumeTracing]]) { - def getSkeleton(tracingId: String): SkeletonTracing = ??? + + def getSkeleton(tracingId: String): Box[SkeletonTracing] = + for { + tracingEither <- tracingsById.get(tracingId) + skeletonTracing <- tracingEither match { + case Left(st: SkeletonTracing) => Full(st) + case _ => Failure(f"Tried to access tracing $tracingId as skeleton, but is volume") + } + } yield skeletonTracing + + def getVolume(tracingId: String): Box[VolumeTracing] = + for { + tracingEither <- tracingsById.get(tracingId) + volumeTracing <- tracingEither match { + case Right(vt: VolumeTracing) => Full(vt) + case _ => Failure(f"Tried to access tracing $tracingId as volume, but is skeleton") + } + } yield volumeTracing def version: Long = annotation.version @@ -59,21 +76,13 @@ case class AnnotationWithTracings(annotation: AnnotationProto, def applySkeletonAction(a: SkeletonUpdateAction)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { - skeletonTracingEither <- tracingsById.get(a.actionTracingId).toFox - skeletonTracing <- skeletonTracingEither match { - case Left(st: SkeletonTracing) => Fox.successful(st) - case _ => Fox.failure("wrong tracing type") - } + skeletonTracing <- getSkeleton(a.actionTracingId) updated = a.applyOn(skeletonTracing) } yield AnnotationWithTracings(annotation, tracingsById.updated(a.actionTracingId, Left(updated))) def applyVolumeAction(a: ApplyableVolumeUpdateAction)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { - volumeTracingEither <- tracingsById.get(a.actionTracingId).toFox - volumeTracing <- volumeTracingEither match { - case Right(vt: VolumeTracing) => Fox.successful(vt) - case _ => Fox.failure("wrong tracing type") - } + volumeTracing <- getVolume(a.actionTracingId) updated = a.applyOn(volumeTracing) } yield AnnotationWithTracings(annotation, tracingsById.updated(a.actionTracingId, Right(updated))) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 792a893d3d6..e13f66a5bc7 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -144,27 +144,14 @@ class VolumeTracingService @Inject()( Fox.failure("Cannot mutate volume data in annotation with editable mapping.") } else updateBucket(tracingId, tracing, a, segmentIndexBuffer, updateGroup.version) ?~> "Failed to save volume data." - case a: UpdateTracingVolumeAction => - Fox.successful( - tracing.copy( - activeSegmentId = Some(a.activeSegmentId), - editPosition = a.editPosition, - editRotation = a.editRotation, - largestSegmentId = a.largestSegmentId, - zoomLevel = a.zoomLevel, - editPositionAdditionalCoordinates = - AdditionalCoordinate.toProto(a.editPositionAdditionalCoordinates) - )) - case a: RevertToVersionVolumeAction => - revertToVolumeVersion(tracingId, a.sourceVersion, updateGroup.version, tracing, userToken) + //case a: RevertToVersionVolumeAction => revertToVolumeVersion(tracingId, a.sourceVersion, updateGroup.version, tracing, userToken) case a: DeleteSegmentDataVolumeAction => if (!tracing.getHasSegmentIndex) { Fox.failure("Cannot delete segment data for annotations without segment index.") } else deleteSegmentData(tracingId, tracing, a, segmentIndexBuffer, updateGroup.version, userToken) ?~> "Failed to delete segment data." - case _: UpdateTdCameraVolumeAction => Fox.successful(tracing) - case a: ApplyableVolumeUpdateAction => Fox.successful(applyUpdateOn(tracing, a)) - case _ => Fox.failure("Unknown action.") + case a: ApplyableVolumeUpdateAction => Fox.successful(applyUpdateOn(tracing, a)) + case _ => Fox.failure("Unknown action.") } case Empty => Fox.empty diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala index a4b3ef91ac5..763dbbbd02f 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala @@ -64,26 +64,24 @@ case class UpdateTracingVolumeAction( actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None -) extends VolumeUpdateAction { +) extends ApplyableVolumeUpdateAction + with ProtoGeometryImplicits { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) override def isViewOnlyChange: Boolean = true -} - -case class RevertToVersionVolumeAction(sourceVersion: Long, - actionTracingId: String, - actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) - extends VolumeUpdateAction { - override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = - this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def applyOn(tracing: VolumeTracing): VolumeTracing = + tracing.copy( + activeSegmentId = Some(activeSegmentId), + editPosition = editPosition, + editRotation = editRotation, + largestSegmentId = largestSegmentId, + zoomLevel = zoomLevel, + editPositionAdditionalCoordinates = AdditionalCoordinate.toProto(editPositionAdditionalCoordinates) + ) } case class UpdateUserBoundingBoxesVolumeAction(boundingBoxes: List[NamedBoundingBox], @@ -178,7 +176,7 @@ case class UpdateTdCameraVolumeAction(actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends VolumeUpdateAction { + extends ApplyableVolumeUpdateAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -186,6 +184,9 @@ case class UpdateTdCameraVolumeAction(actionTracingId: String, this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def applyOn(tracing: VolumeTracing): VolumeTracing = + tracing + override def isViewOnlyChange: Boolean = true } @@ -262,7 +263,7 @@ case class DeleteSegmentVolumeAction(id: Long, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends VolumeUpdateAction { + extends ApplyableVolumeUpdateAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -270,8 +271,8 @@ case class DeleteSegmentVolumeAction(id: Long, this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - /*override def applyOn(tracing: VolumeTracing): VolumeTracing = - tracing.withSegments(tracing.segments.filter(_.segmentId != id))*/ + override def applyOn(tracing: VolumeTracing): VolumeTracing = + tracing.withSegments(tracing.segments.filter(_.segmentId != id)) } @@ -324,6 +325,7 @@ case class UpdateSegmentGroupsVolumeAction(segmentGroups: List[UpdateActionSegme override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) } +// TODO this now exists only for UpdateBucket. Make it a slimmed down version of that rather than generic? case class CompactVolumeUpdateAction(name: String, value: JsObject, actionTracingId: String, @@ -368,9 +370,6 @@ object UpdateBucketVolumeAction { object UpdateTracingVolumeAction { implicit val jsonFormat: OFormat[UpdateTracingVolumeAction] = Json.format[UpdateTracingVolumeAction] } -object RevertToVersionVolumeAction { - implicit val jsonFormat: OFormat[RevertToVersionVolumeAction] = Json.format[RevertToVersionVolumeAction] -} object UpdateUserBoundingBoxesVolumeAction { implicit val jsonFormat: OFormat[UpdateUserBoundingBoxesVolumeAction] = Json.format[UpdateUserBoundingBoxesVolumeAction] From a9cd852b0c91a8efcc9b03a0f61a64206c662fea Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 30 Jul 2024 13:11:17 +0200 Subject: [PATCH 030/150] apply bucketMutating actions --- .../annotation/TSAnnotationService.scala | 26 +++++-- .../volume/VolumeTracingService.scala | 72 ++++++++----------- .../tracings/volume/VolumeUpdateActions.scala | 6 +- 3 files changed, 53 insertions(+), 51 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 626d6519141..460efe755ea 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -15,7 +15,9 @@ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{ } import com.scalableminds.webknossos.tracingstore.tracings.volume.{ ApplyableVolumeUpdateAction, + BucketMutatingVolumeUpdateAction, UpdateBucketVolumeAction, + VolumeTracingService, VolumeUpdateAction } import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} @@ -88,7 +90,8 @@ case class AnnotationWithTracings(annotation: AnnotationProto, } class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosClient, - tracingDataStore: TracingDataStore) + tracingDataStore: TracingDataStore, + volumeTracingService: VolumeTracingService) extends KeyValueStoreImplicits { def reportUpdates(annotationId: String, updateGroups: List[UpdateActionGroup], userToken: Option[String]): Fox[Unit] = @@ -109,11 +112,22 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl def handleUpdateGroup(annotationId: String, updateActionGroup: UpdateActionGroup, previousVersion: Long, - userToken: Option[String]): Fox[Unit] = - // TODO apply some updates directly? transform to compact? - tracingDataStore.annotationUpdates.put(annotationId, - updateActionGroup.version, - preprocessActionsForStorage(updateActionGroup)) + userToken: Option[String])(implicit ec: ExecutionContext): Fox[Unit] = + for { + _ <- tracingDataStore.annotationUpdates.put(annotationId, + updateActionGroup.version, + preprocessActionsForStorage(updateActionGroup)) + bucketMutatingActions = findBucketMutatingActions(updateActionGroup) + _ <- Fox.runIf(bucketMutatingActions.nonEmpty)( + volumeTracingService + .applyBucketMutatingActions(bucketMutatingActions, previousVersion, updateActionGroup.version, userToken)) + } yield () + + private def findBucketMutatingActions(updateActionGroup: UpdateActionGroup): List[BucketMutatingVolumeUpdateAction] = + updateActionGroup.actions.flatMap { + case a: BucketMutatingVolumeUpdateAction => Some(a) + case _ => None + } private def preprocessActionsForStorage(updateActionGroup: UpdateActionGroup): List[UpdateAction] = { val actionsWithInfo = updateActionGroup.actions.map( diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index e13f66a5bc7..0569a4c261c 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -116,55 +116,41 @@ class VolumeTracingService @Inject()( private def applyUpdateOn(tracing: VolumeTracing, update: ApplyableVolumeUpdateAction): VolumeTracing = ??? - def handleUpdateGroup(tracingId: String, - updateGroup: UpdateActionGroup, - previousVersion: Long, - userToken: Option[String]): Fox[Unit] = + def applyBucketMutatingActions(updateActions: List[BucketMutatingVolumeUpdateAction], + previousVersion: Long, + newVersion: Long, + userToken: Option[String]): Fox[Unit] = for { // warning, may be called multiple times with the same version number (due to transaction management). // frontend ensures that each bucket is only updated once per transaction - fallbackLayer <- getFallbackLayer(tracingId) + tracingId <- updateActions.headOption.map(_.actionTracingId).toFox + fallbackLayerOpt <- getFallbackLayer(tracingId) tracing <- find(tracingId) ?~> "tracing.notFound" - segmentIndexBuffer <- Fox.successful( - new VolumeSegmentIndexBuffer( - tracingId, - volumeSegmentIndexClient, - updateGroup.version, - remoteDatastoreClient, - fallbackLayer, - AdditionalAxis.fromProtosAsOpt(tracing.additionalAxes), - userToken - )) - updatedTracing: VolumeTracing <- updateGroup.actions.foldLeft(find(tracingId)) { (tracingFox, action) => - tracingFox.futureBox.flatMap { - case Full(tracing) => - action match { - case a: UpdateBucketVolumeAction => - if (tracing.getMappingIsEditable) { - Fox.failure("Cannot mutate volume data in annotation with editable mapping.") - } else - updateBucket(tracingId, tracing, a, segmentIndexBuffer, updateGroup.version) ?~> "Failed to save volume data." - //case a: RevertToVersionVolumeAction => revertToVolumeVersion(tracingId, a.sourceVersion, updateGroup.version, tracing, userToken) - case a: DeleteSegmentDataVolumeAction => - if (!tracing.getHasSegmentIndex) { - Fox.failure("Cannot delete segment data for annotations without segment index.") - } else - deleteSegmentData(tracingId, tracing, a, segmentIndexBuffer, updateGroup.version, userToken) ?~> "Failed to delete segment data." - case a: ApplyableVolumeUpdateAction => Fox.successful(applyUpdateOn(tracing, a)) - case _ => Fox.failure("Unknown action.") - } - case Empty => - Fox.empty - case f: Failure => - f.toFox - } + segmentIndexBuffer = new VolumeSegmentIndexBuffer( + tracingId, + volumeSegmentIndexClient, + newVersion, + remoteDatastoreClient, + fallbackLayerOpt, + AdditionalAxis.fromProtosAsOpt(tracing.additionalAxes), + userToken + ) + _ <- Fox.serialCombined(updateActions) { + case a: UpdateBucketVolumeAction => + if (tracing.getMappingIsEditable) { + Fox.failure("Cannot mutate volume data in annotation with editable mapping.") + } else + updateBucket(tracingId, tracing, a, segmentIndexBuffer, newVersion) ?~> "Failed to save volume data." + //case a: RevertToVersionVolumeAction => revertToVolumeVersion(tracingId, a.sourceVersion, updateGroup.version, tracing, userToken) + case a: DeleteSegmentDataVolumeAction => + if (!tracing.getHasSegmentIndex) { + Fox.failure("Cannot delete segment data for annotations without segment index.") + } else + deleteSegmentData(tracingId, tracing, a, segmentIndexBuffer, newVersion, userToken) ?~> "Failed to delete segment data." + case _ => Fox.failure("Unknown bucket-mutating action.") } _ <- segmentIndexBuffer.flush() - _ <- save(updatedTracing.copy(version = updateGroup.version), Some(tracingId), updateGroup.version) - _ <- tracingDataStore.volumeUpdates.put(tracingId, - updateGroup.version, - updateGroup.actions.map(_.addTimestamp(updateGroup.timestamp))) - } yield Fox.successful(()) + } yield () private def updateBucket(tracingId: String, volumeTracing: VolumeTracing, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala index 763dbbbd02f..f5230ec79a3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala @@ -32,6 +32,8 @@ trait ApplyableVolumeUpdateAction extends VolumeUpdateAction { def applyOn(tracing: VolumeTracing): VolumeTracing } +trait BucketMutatingVolumeUpdateAction extends VolumeUpdateAction + case class UpdateBucketVolumeAction(position: Vec3Int, cubeSize: Int, mag: Vec3Int, @@ -41,7 +43,7 @@ case class UpdateBucketVolumeAction(position: Vec3Int, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends VolumeUpdateAction { + extends BucketMutatingVolumeUpdateAction { lazy val data: Array[Byte] = Base64.getDecoder.decode(base64Data) override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -263,7 +265,7 @@ case class DeleteSegmentVolumeAction(id: Long, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends ApplyableVolumeUpdateAction { + extends BucketMutatingVolumeUpdateAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) From 5ae6f466eca64264c648cf312255145e4f36980c Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 30 Jul 2024 13:20:31 +0200 Subject: [PATCH 031/150] fixes --- .../annotation/AnnotationTransactionService.scala | 12 +++++------- .../annotation/TSAnnotationService.scala | 9 +++------ .../controllers/VolumeTracingController.scala | 4 ++-- .../tracings/volume/VolumeTracingService.scala | 8 ++------ .../tracings/volume/VolumeUpdateActions.scala | 3 ++- 5 files changed, 14 insertions(+), 22 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala index 3c7f5f48354..c993dd9bfaf 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala @@ -117,9 +117,8 @@ class AnnotationTransactionService @Inject()( transactionGroupIndex: Int): Fox[Boolean] = handledGroupIdStore.contains(handledGroupKey(annotationId, transactionId, version, transactionGroupIndex)) - private def concatenateUpdateGroupsOfTransaction( - previousActionGroups: List[UpdateActionGroup], - lastActionGroup: UpdateActionGroup): UpdateActionGroup = + private def concatenateUpdateGroupsOfTransaction(previousActionGroups: List[UpdateActionGroup], + lastActionGroup: UpdateActionGroup): UpdateActionGroup = if (previousActionGroups.isEmpty) lastActionGroup else { val allActionGroups = previousActionGroups :+ lastActionGroup @@ -148,9 +147,8 @@ class AnnotationTransactionService @Inject()( } // Perform version check and commit the passed updates - private def commitUpdates(annotationId: String, - updateGroups: List[UpdateActionGroup], - userToken: Option[String])(implicit ec: ExecutionContext): Fox[Long] = + private def commitUpdates(annotationId: String, updateGroups: List[UpdateActionGroup], userToken: Option[String])( + implicit ec: ExecutionContext): Fox[Long] = for { _ <- annotationService.reportUpdates(annotationId, updateGroups, userToken) currentCommittedVersion: Fox[Long] = annotationService.currentVersion(annotationId) @@ -158,7 +156,7 @@ class AnnotationTransactionService @Inject()( previousVersion.flatMap { prevVersion: Long => if (prevVersion + 1 == updateGroup.version) { for { - _ <- annotationService.handleUpdateGroup(annotationId, updateGroup, prevVersion, userToken) + _ <- annotationService.handleUpdateGroup(annotationId, updateGroup, userToken) _ <- saveToHandledGroupIdStore(annotationId, updateGroup.transactionId, updateGroup.version, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 460efe755ea..ef08058b404 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -109,18 +109,15 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl def currentVersion(annotationId: String): Fox[Long] = ??? - def handleUpdateGroup(annotationId: String, - updateActionGroup: UpdateActionGroup, - previousVersion: Long, - userToken: Option[String])(implicit ec: ExecutionContext): Fox[Unit] = + def handleUpdateGroup(annotationId: String, updateActionGroup: UpdateActionGroup, userToken: Option[String])( + implicit ec: ExecutionContext): Fox[Unit] = for { _ <- tracingDataStore.annotationUpdates.put(annotationId, updateActionGroup.version, preprocessActionsForStorage(updateActionGroup)) bucketMutatingActions = findBucketMutatingActions(updateActionGroup) _ <- Fox.runIf(bucketMutatingActions.nonEmpty)( - volumeTracingService - .applyBucketMutatingActions(bucketMutatingActions, previousVersion, updateActionGroup.version, userToken)) + volumeTracingService.applyBucketMutatingActions(bucketMutatingActions, updateActionGroup.version, userToken)) } yield () private def findBucketMutatingActions(updateActionGroup: UpdateActionGroup): List[BucketMutatingVolumeUpdateAction] = diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 0c733f0ed9a..160036b7c7f 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -373,7 +373,7 @@ class VolumeTracingController @Inject()( isLocked = Some(true), actionTracingId = tracingId, actionTimestamp = Some(System.currentTimeMillis())) - _ <- tracingService.handleUpdateGroup( + /*_ <- tracingService.handleUpdateGroup( // TODO tracingId, UpdateActionGroup(tracing.version + 1, System.currentTimeMillis(), @@ -386,7 +386,7 @@ class VolumeTracingController @Inject()( 0), tracing.version, urlOrHeaderToken(token, request) - ) + )*/ infoJson <- editableMappingService.infoJson(tracingId = tracingId, editableMappingId = editableMappingId, editableMappingInfo = editableMappingInfo, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 0569a4c261c..c69c4c90615 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -15,7 +15,6 @@ import com.scalableminds.webknossos.datastore.models.datasource.{AdditionalAxis, import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing.ElementClassProto import com.scalableminds.webknossos.datastore.models.requests.DataServiceDataRequest import com.scalableminds.webknossos.datastore.models.{ - AdditionalCoordinate, BucketPosition, UnsignedInteger, UnsignedIntegerArray, @@ -114,10 +113,7 @@ class VolumeTracingService @Inject()( mappingName, editableMappingTracingId) ?~> "volumeSegmentIndex.update.failed" - private def applyUpdateOn(tracing: VolumeTracing, update: ApplyableVolumeUpdateAction): VolumeTracing = ??? - def applyBucketMutatingActions(updateActions: List[BucketMutatingVolumeUpdateAction], - previousVersion: Long, newVersion: Long, userToken: Option[String]): Fox[Unit] = for { @@ -888,7 +884,7 @@ class VolumeTracingService @Inject()( 1, 0 ) - _ <- Fox.runIf(!dryRun)(handleUpdateGroup(tracingId, updateGroup, tracing.version, userToken)) + // TODO _ <- Fox.runIf(!dryRun)(handleUpdateGroup(tracingId, updateGroup, tracing.version, userToken)) } yield Some(processedBucketCount) } @@ -975,7 +971,7 @@ class VolumeTracingService @Inject()( 1, 0 ) - _ <- handleUpdateGroup(tracingId, updateGroup, tracing.version, userToken) + // TODO: _ <- handleUpdateGroup(tracingId, updateGroup, tracing.version, userToken) } yield mergedVolume.largestSegmentId.toPositiveLong } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala index f5230ec79a3..f12dd1a8948 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala @@ -265,7 +265,8 @@ case class DeleteSegmentVolumeAction(id: Long, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends BucketMutatingVolumeUpdateAction { + extends BucketMutatingVolumeUpdateAction + with ApplyableVolumeUpdateAction { // TODO double-check that it is matched against both traits override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) From 7b3d7c25d1565c3d4e633ea921173d0d226bbf1d Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 30 Jul 2024 13:36:21 +0200 Subject: [PATCH 032/150] small cleanup --- .../annotation/AnnotationWithTracings.scala | 73 +++++++++++++++++++ .../annotation/TSAnnotationService.scala | 63 ---------------- .../tracings/volume/VolumeUpdateActions.scala | 2 +- 3 files changed, 74 insertions(+), 64 deletions(-) create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala new file mode 100644 index 00000000000..ce13e61445a --- /dev/null +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -0,0 +1,73 @@ +package com.scalableminds.webknossos.tracingstore.annotation + +import com.scalableminds.util.tools.Fox +import com.scalableminds.webknossos.datastore.Annotation.{AnnotationLayerProto, AnnotationProto} +import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing +import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing +import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType +import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.SkeletonUpdateAction +import com.scalableminds.webknossos.tracingstore.tracings.volume.ApplyableVolumeUpdateAction +import net.liftweb.common.{Box, Failure, Full} + +import scala.concurrent.ExecutionContext + +case class AnnotationWithTracings(annotation: AnnotationProto, + tracingsById: Map[String, Either[SkeletonTracing, VolumeTracing]]) { + + def getSkeleton(tracingId: String): Box[SkeletonTracing] = + for { + tracingEither <- tracingsById.get(tracingId) + skeletonTracing <- tracingEither match { + case Left(st: SkeletonTracing) => Full(st) + case _ => Failure(f"Tried to access tracing $tracingId as skeleton, but is volume") + } + } yield skeletonTracing + + def getVolume(tracingId: String): Box[VolumeTracing] = + for { + tracingEither <- tracingsById.get(tracingId) + volumeTracing <- tracingEither match { + case Right(vt: VolumeTracing) => Full(vt) + case _ => Failure(f"Tried to access tracing $tracingId as volume, but is skeleton") + } + } yield volumeTracing + + def version: Long = annotation.version + + def addTracing(a: AddLayerAnnotationUpdateAction): AnnotationWithTracings = + AnnotationWithTracings( + annotation.copy( + layers = annotation.layers :+ AnnotationLayerProto(a.tracingId, + a.layerName, + `type` = AnnotationLayerType.toProto(a.`type`))), + tracingsById) + + def deleteTracing(a: DeleteLayerAnnotationUpdateAction): AnnotationWithTracings = + AnnotationWithTracings(annotation.copy(layers = annotation.layers.filter(_.tracingId != a.tracingId)), tracingsById) + + def updateLayerMetadata(a: UpdateLayerMetadataAnnotationUpdateAction): AnnotationWithTracings = + AnnotationWithTracings(annotation.copy(layers = annotation.layers.map(l => + if (l.tracingId == a.tracingId) l.copy(name = a.layerName) else l)), + tracingsById) + + def updateMetadata(a: UpdateMetadataAnnotationUpdateAction): AnnotationWithTracings = + AnnotationWithTracings(annotation.copy(name = a.name, description = a.description), tracingsById) + + def incrementVersion: AnnotationWithTracings = + AnnotationWithTracings(annotation.copy(version = annotation.version + 1L), tracingsById) + + def withVersion(newVersion: Long): AnnotationWithTracings = + AnnotationWithTracings(annotation.copy(version = newVersion), tracingsById) // TODO also update version in tracings? + + def applySkeletonAction(a: SkeletonUpdateAction)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = + for { + skeletonTracing <- getSkeleton(a.actionTracingId) + updated = a.applyOn(skeletonTracing) + } yield AnnotationWithTracings(annotation, tracingsById.updated(a.actionTracingId, Left(updated))) + + def applyVolumeAction(a: ApplyableVolumeUpdateAction)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = + for { + volumeTracing <- getVolume(a.actionTracingId) + updated = a.applyOn(volumeTracing) + } yield AnnotationWithTracings(annotation, tracingsById.updated(a.actionTracingId, Right(updated))) +} diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index ef08058b404..31ff46b3bb4 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -28,67 +28,6 @@ import play.api.libs.json.{JsObject, JsValue, Json} import javax.inject.Inject import scala.concurrent.ExecutionContext -case class AnnotationWithTracings(annotation: AnnotationProto, - tracingsById: Map[String, Either[SkeletonTracing, VolumeTracing]]) { - - def getSkeleton(tracingId: String): Box[SkeletonTracing] = - for { - tracingEither <- tracingsById.get(tracingId) - skeletonTracing <- tracingEither match { - case Left(st: SkeletonTracing) => Full(st) - case _ => Failure(f"Tried to access tracing $tracingId as skeleton, but is volume") - } - } yield skeletonTracing - - def getVolume(tracingId: String): Box[VolumeTracing] = - for { - tracingEither <- tracingsById.get(tracingId) - volumeTracing <- tracingEither match { - case Right(vt: VolumeTracing) => Full(vt) - case _ => Failure(f"Tried to access tracing $tracingId as volume, but is skeleton") - } - } yield volumeTracing - - def version: Long = annotation.version - - def addTracing(a: AddLayerAnnotationUpdateAction): AnnotationWithTracings = - AnnotationWithTracings( - annotation.copy( - layers = annotation.layers :+ AnnotationLayerProto(a.tracingId, - a.layerName, - `type` = AnnotationLayerType.toProto(a.`type`))), - tracingsById) - - def deleteTracing(a: DeleteLayerAnnotationUpdateAction): AnnotationWithTracings = - AnnotationWithTracings(annotation.copy(layers = annotation.layers.filter(_.tracingId != a.tracingId)), tracingsById) - - def updateLayerMetadata(a: UpdateLayerMetadataAnnotationUpdateAction): AnnotationWithTracings = - AnnotationWithTracings(annotation.copy(layers = annotation.layers.map(l => - if (l.tracingId == a.tracingId) l.copy(name = a.layerName) else l)), - tracingsById) - - def updateMetadata(a: UpdateMetadataAnnotationUpdateAction): AnnotationWithTracings = - AnnotationWithTracings(annotation.copy(name = a.name, description = a.description), tracingsById) - - def incrementVersion: AnnotationWithTracings = - AnnotationWithTracings(annotation.copy(version = annotation.version + 1L), tracingsById) - - def withVersion(newVersion: Long): AnnotationWithTracings = - AnnotationWithTracings(annotation.copy(version = newVersion), tracingsById) // TODO also update version in tracings? - - def applySkeletonAction(a: SkeletonUpdateAction)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = - for { - skeletonTracing <- getSkeleton(a.actionTracingId) - updated = a.applyOn(skeletonTracing) - } yield AnnotationWithTracings(annotation, tracingsById.updated(a.actionTracingId, Left(updated))) - - def applyVolumeAction(a: ApplyableVolumeUpdateAction)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = - for { - volumeTracing <- getVolume(a.actionTracingId) - updated = a.applyOn(volumeTracing) - } yield AnnotationWithTracings(annotation, tracingsById.updated(a.actionTracingId, Right(updated))) -} - class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosClient, tracingDataStore: TracingDataStore, volumeTracingService: VolumeTracingService) @@ -168,8 +107,6 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl annotationWithTracings.applySkeletonAction(a) case a: ApplyableVolumeUpdateAction => annotationWithTracings.applyVolumeAction(a) - case a: VolumeUpdateAction => - Fox.successful(annotationWithTracings) // TODO case _ => Fox.failure("Received unsupported AnnotationUpdateAction action") } } yield updated diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala index f12dd1a8948..ad2e3eab783 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala @@ -266,7 +266,7 @@ case class DeleteSegmentVolumeAction(id: Long, actionAuthorId: Option[String] = None, info: Option[String] = None) extends BucketMutatingVolumeUpdateAction - with ApplyableVolumeUpdateAction { // TODO double-check that it is matched against both traits + with ApplyableVolumeUpdateAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) From 62e671dc6f8077b2879e4a99e10971c7d1ea9ecb Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 21 Aug 2024 11:29:01 +0200 Subject: [PATCH 033/150] WIP: also maintain editable mappings when applying updates --- .../AnnotationTransactionService.scala | 4 +- .../annotation/AnnotationWithTracings.scala | 39 ++++++++++++++----- .../annotation/TSAnnotationService.scala | 19 ++++++--- 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala index c993dd9bfaf..e0b8fd651de 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala @@ -140,7 +140,7 @@ class AnnotationTransactionService @Inject()( if (updateGroups.forall(_.transactionGroupCount == 1)) { commitUpdates(annotationId, updateGroups, userToken) } else { - updateGroups.foldLeft(annotationService.currentVersion(annotationId)) { + updateGroups.foldLeft(annotationService.currentMaterializableVersion(annotationId)) { (currentCommittedVersionFox, updateGroup) => handleUpdateGroupForTransaction(annotationId, currentCommittedVersionFox, updateGroup, userToken) } @@ -151,7 +151,7 @@ class AnnotationTransactionService @Inject()( implicit ec: ExecutionContext): Fox[Long] = for { _ <- annotationService.reportUpdates(annotationId, updateGroups, userToken) - currentCommittedVersion: Fox[Long] = annotationService.currentVersion(annotationId) + currentCommittedVersion: Fox[Long] = annotationService.currentMaterializableVersion(annotationId) newVersion <- updateGroups.foldLeft(currentCommittedVersion) { (previousVersion, updateGroup) => previousVersion.flatMap { prevVersion: Long => if (prevVersion + 1 == updateGroup.version) { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index ce13e61445a..9469458df93 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -2,17 +2,21 @@ package com.scalableminds.webknossos.tracingstore.annotation import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.Annotation.{AnnotationLayerProto, AnnotationProto} +import com.scalableminds.webknossos.datastore.EditableMappingInfo.EditableMappingInfo import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType +import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingUpdater import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.SkeletonUpdateAction import com.scalableminds.webknossos.tracingstore.tracings.volume.ApplyableVolumeUpdateAction import net.liftweb.common.{Box, Failure, Full} import scala.concurrent.ExecutionContext -case class AnnotationWithTracings(annotation: AnnotationProto, - tracingsById: Map[String, Either[SkeletonTracing, VolumeTracing]]) { +case class AnnotationWithTracings( + annotation: AnnotationProto, + tracingsById: Map[String, Either[SkeletonTracing, VolumeTracing]], + editableMappingsByTracingId: Map[String, (EditableMappingInfo, EditableMappingUpdater)]) { def getSkeleton(tracingId: String): Box[SkeletonTracing] = for { @@ -40,34 +44,49 @@ case class AnnotationWithTracings(annotation: AnnotationProto, layers = annotation.layers :+ AnnotationLayerProto(a.tracingId, a.layerName, `type` = AnnotationLayerType.toProto(a.`type`))), - tracingsById) + tracingsById, + editableMappingsByTracingId + ) def deleteTracing(a: DeleteLayerAnnotationUpdateAction): AnnotationWithTracings = - AnnotationWithTracings(annotation.copy(layers = annotation.layers.filter(_.tracingId != a.tracingId)), tracingsById) + AnnotationWithTracings(annotation.copy(layers = annotation.layers.filter(_.tracingId != a.tracingId)), + tracingsById, + editableMappingsByTracingId) def updateLayerMetadata(a: UpdateLayerMetadataAnnotationUpdateAction): AnnotationWithTracings = AnnotationWithTracings(annotation.copy(layers = annotation.layers.map(l => if (l.tracingId == a.tracingId) l.copy(name = a.layerName) else l)), - tracingsById) + tracingsById, + editableMappingsByTracingId) def updateMetadata(a: UpdateMetadataAnnotationUpdateAction): AnnotationWithTracings = - AnnotationWithTracings(annotation.copy(name = a.name, description = a.description), tracingsById) + AnnotationWithTracings(annotation.copy(name = a.name, description = a.description), + tracingsById, + editableMappingsByTracingId) def incrementVersion: AnnotationWithTracings = - AnnotationWithTracings(annotation.copy(version = annotation.version + 1L), tracingsById) + AnnotationWithTracings(annotation.copy(version = annotation.version + 1L), + tracingsById, + editableMappingsByTracingId) def withVersion(newVersion: Long): AnnotationWithTracings = - AnnotationWithTracings(annotation.copy(version = newVersion), tracingsById) // TODO also update version in tracings? + AnnotationWithTracings(annotation.copy(version = newVersion), tracingsById, editableMappingsByTracingId) // TODO also update version in tracings? def applySkeletonAction(a: SkeletonUpdateAction)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { skeletonTracing <- getSkeleton(a.actionTracingId) updated = a.applyOn(skeletonTracing) - } yield AnnotationWithTracings(annotation, tracingsById.updated(a.actionTracingId, Left(updated))) + } yield + AnnotationWithTracings(annotation, + tracingsById.updated(a.actionTracingId, Left(updated)), + editableMappingsByTracingId) def applyVolumeAction(a: ApplyableVolumeUpdateAction)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { volumeTracing <- getVolume(a.actionTracingId) updated = a.applyOn(volumeTracing) - } yield AnnotationWithTracings(annotation, tracingsById.updated(a.actionTracingId, Right(updated))) + } yield + AnnotationWithTracings(annotation, + tracingsById.updated(a.actionTracingId, Right(updated)), + editableMappingsByTracingId) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 31ff46b3bb4..04dfc10194b 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -3,10 +3,14 @@ package com.scalableminds.webknossos.tracingstore.annotation import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox import com.scalableminds.util.tools.Fox.option2Fox -import com.scalableminds.webknossos.datastore.Annotation.{AnnotationLayerProto, AnnotationProto} +import com.scalableminds.webknossos.datastore.Annotation.AnnotationProto +import com.scalableminds.webknossos.datastore.EditableMappingInfo.EditableMappingInfo import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing -import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType +import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ + EditableMappingUpdateAction, + EditableMappingUpdater +} import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{ CreateNodeSkeletonAction, DeleteNodeSkeletonAction, @@ -22,7 +26,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ } import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingUpdatesReport} -import net.liftweb.common.{Box, Empty, Failure, Full} +import net.liftweb.common.{Empty, Full} import play.api.libs.json.{JsObject, JsValue, Json} import javax.inject.Inject @@ -46,7 +50,8 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl )) } yield () - def currentVersion(annotationId: String): Fox[Long] = ??? + def currentMaterializableVersion(annotationId: String): Fox[Long] = + tracingDataStore.annotations.getVersion(annotationId, mayBeEmpty = Some(true), emptyFallback = Some(0L)) def handleUpdateGroup(annotationId: String, updateActionGroup: UpdateActionGroup, userToken: Option[String])( implicit ec: ExecutionContext): Fox[Unit] = @@ -107,6 +112,8 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl annotationWithTracings.applySkeletonAction(a) case a: ApplyableVolumeUpdateAction => annotationWithTracings.applyVolumeAction(a) + case a: EditableMappingUpdateAction => + Fox.failure("not yet implemented") case _ => Fox.failure("Received unsupported AnnotationUpdateAction action") } } yield updated @@ -157,6 +164,8 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl case u: VolumeUpdateAction => Some(u.actionTracingId) case _ => None } + // TODO fetch editable mappings + instantiate editableMappingUpdaters/buffers if there are updates for them + val editableMappingsMap: Map[String, (EditableMappingInfo, EditableMappingUpdater)] = Map.empty for { skeletonTracings <- Fox.serialCombined(skeletonTracingIds)( id => @@ -172,7 +181,7 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl volumeTracingsMap: Map[String, Either[SkeletonTracing, VolumeTracing]] = volumeTracingIds .zip(volumeTracings.map(versioned => Right[SkeletonTracing, VolumeTracing](versioned.value))) .toMap - } yield AnnotationWithTracings(annotation, skeletonTracingsMap ++ volumeTracingsMap) + } yield AnnotationWithTracings(annotation, skeletonTracingsMap ++ volumeTracingsMap, editableMappingsMap) } private def applyUpdates(annotation: AnnotationWithTracings, From c4dc9577116c148d68fcc167375e2e84b1dc8119 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 21 Aug 2024 13:57:44 +0200 Subject: [PATCH 034/150] WIP: createTracing --- app/controllers/AnnotationController.scala | 19 ++----------------- app/models/annotation/AnnotationService.scala | 2 +- .../annotation/AnnotationUpdateActions.scala | 18 ++++++++++++++++-- .../annotation/AnnotationWithTracings.scala | 9 ++++++++- .../annotation/TSAnnotationService.scala | 10 ++++++++-- 5 files changed, 35 insertions(+), 23 deletions(-) diff --git a/app/controllers/AnnotationController.scala b/app/controllers/AnnotationController.scala index a9497011191..b3160e1f665 100755 --- a/app/controllers/AnnotationController.scala +++ b/app/controllers/AnnotationController.scala @@ -7,13 +7,10 @@ import com.scalableminds.util.geometry.BoundingBox import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType.AnnotationLayerType -import com.scalableminds.webknossos.datastore.models.annotation.{ - AnnotationLayer, - AnnotationLayerStatistics, - AnnotationLayerType -} +import com.scalableminds.webknossos.datastore.models.annotation.{AnnotationLayer, AnnotationLayerStatistics, AnnotationLayerType} import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis import com.scalableminds.webknossos.datastore.rpc.RPC +import com.scalableminds.webknossos.tracingstore.annotation.AnnotationLayerParameters import com.scalableminds.webknossos.tracingstore.tracings.volume.ResolutionRestrictions import com.scalableminds.webknossos.tracingstore.tracings.{TracingIds, TracingType} import mail.{MailchimpClient, MailchimpTag} @@ -40,18 +37,6 @@ import javax.inject.Inject import scala.concurrent.ExecutionContext import scala.concurrent.duration._ -case class AnnotationLayerParameters(typ: AnnotationLayerType, - fallbackLayerName: Option[String], - autoFallbackLayer: Boolean = false, - mappingName: Option[String] = None, - resolutionRestrictions: Option[ResolutionRestrictions], - name: Option[String], - additionalAxes: Option[Seq[AdditionalAxis]]) -object AnnotationLayerParameters { - implicit val jsonFormat: OFormat[AnnotationLayerParameters] = - Json.using[WithDefaultValues].format[AnnotationLayerParameters] -} - class AnnotationController @Inject()( annotationDAO: AnnotationDAO, annotationLayerDAO: AnnotationLayerDAO, diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index 651473ba19d..c84a095d99f 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -32,6 +32,7 @@ import com.scalableminds.webknossos.datastore.models.datasource.{ SegmentationLayerLike => SegmentationLayer } import com.scalableminds.webknossos.datastore.rpc.RPC +import com.scalableminds.webknossos.tracingstore.annotation.AnnotationLayerParameters import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeDataZipFormat.VolumeDataZipFormat import com.scalableminds.webknossos.tracingstore.tracings.volume.{ @@ -41,7 +42,6 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ VolumeTracingDownsampling } import com.typesafe.scalalogging.LazyLogging -import controllers.AnnotationLayerParameters import models.annotation.AnnotationState._ import models.annotation.AnnotationType.AnnotationType import models.annotation.handler.SavedTracingInformationHandler diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala index e9fa3268163..860cb4f81b9 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala @@ -1,13 +1,27 @@ package com.scalableminds.webknossos.tracingstore.annotation import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType.AnnotationLayerType +import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis +import com.scalableminds.webknossos.tracingstore.tracings.volume.ResolutionRestrictions +import play.api.libs.json.Json.WithDefaultValues import play.api.libs.json.{Json, OFormat} +case class AnnotationLayerParameters(typ: AnnotationLayerType, + fallbackLayerName: Option[String], + autoFallbackLayer: Boolean = false, + mappingName: Option[String] = None, + resolutionRestrictions: Option[ResolutionRestrictions], + name: Option[String], + additionalAxes: Option[Seq[AdditionalAxis]]) +object AnnotationLayerParameters { + implicit val jsonFormat: OFormat[AnnotationLayerParameters] = + Json.using[WithDefaultValues].format[AnnotationLayerParameters] +} + trait AnnotationUpdateAction extends UpdateAction -case class AddLayerAnnotationUpdateAction(layerName: String, +case class AddLayerAnnotationUpdateAction(layerParameters: AnnotationLayerParameters, tracingId: String, - `type`: AnnotationLayerType, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index 9469458df93..dcf07d1d5f8 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -6,7 +6,10 @@ import com.scalableminds.webknossos.datastore.EditableMappingInfo.EditableMappin import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType -import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingUpdater +import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ + EditableMappingUpdateAction, + EditableMappingUpdater +} import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.SkeletonUpdateAction import com.scalableminds.webknossos.tracingstore.tracings.volume.ApplyableVolumeUpdateAction import net.liftweb.common.{Box, Failure, Full} @@ -89,4 +92,8 @@ case class AnnotationWithTracings( AnnotationWithTracings(annotation, tracingsById.updated(a.actionTracingId, Right(updated)), editableMappingsByTracingId) + + def applyEditableMappingAction(a: EditableMappingUpdateAction)( + implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = + Fox.failure("not implemented yet") // TODO } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 04dfc10194b..c972d51358f 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -100,7 +100,6 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl for { updated <- updateAction match { case a: AddLayerAnnotationUpdateAction => - // TODO create tracing object (ask wk for needed parameters e.g. fallback layer info?) Fox.successful(annotationWithTracings.addTracing(a)) case a: DeleteLayerAnnotationUpdateAction => Fox.successful(annotationWithTracings.deleteTracing(a)) @@ -113,11 +112,18 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl case a: ApplyableVolumeUpdateAction => annotationWithTracings.applyVolumeAction(a) case a: EditableMappingUpdateAction => - Fox.failure("not yet implemented") + annotationWithTracings.applyEditableMappingAction(a) + // TODO make Mapping Editable + // Note: UpdateBucketVolumeActions are not handled here, but instead eagerly on saving. case _ => Fox.failure("Received unsupported AnnotationUpdateAction action") } } yield updated + def createTracing(a: AddLayerAnnotationUpdateAction)( + implicit ec: ExecutionContext): Fox[Either[SkeletonTracing, VolumeTracing]] = + Fox.failure("not implemented") + // TODO create tracing object (ask wk for needed parameters e.g. fallback layer info?) + def updateActionLog(annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]): Fox[JsValue] = { def versionedTupleToJson(tuple: (Long, List[UpdateAction])): JsObject = Json.obj( From ff579f86639f8244ab341b78ced6ba5fd31afbd0 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 22 Aug 2024 14:02:28 +0200 Subject: [PATCH 035/150] adapt tests, imports --- app/controllers/AnnotationController.scala | 10 ++-- app/controllers/LegacyApiController.scala | 1 + test/backend/Dummies.scala | 2 + .../SkeletonUpdateActionsUnitTestSuite.scala | 48 ++++++++++++------- .../VolumeUpdateActionsUnitTestSuite.scala | 23 ++++++--- .../annotation/AnnotationWithTracings.scala | 10 ++-- 6 files changed, 61 insertions(+), 33 deletions(-) diff --git a/app/controllers/AnnotationController.scala b/app/controllers/AnnotationController.scala index b3160e1f665..222a3d3a21b 100755 --- a/app/controllers/AnnotationController.scala +++ b/app/controllers/AnnotationController.scala @@ -6,12 +6,13 @@ import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContex import com.scalableminds.util.geometry.BoundingBox import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.{Fox, FoxImplicits} -import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType.AnnotationLayerType -import com.scalableminds.webknossos.datastore.models.annotation.{AnnotationLayer, AnnotationLayerStatistics, AnnotationLayerType} -import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis +import com.scalableminds.webknossos.datastore.models.annotation.{ + AnnotationLayer, + AnnotationLayerStatistics, + AnnotationLayerType +} import com.scalableminds.webknossos.datastore.rpc.RPC import com.scalableminds.webknossos.tracingstore.annotation.AnnotationLayerParameters -import com.scalableminds.webknossos.tracingstore.tracings.volume.ResolutionRestrictions import com.scalableminds.webknossos.tracingstore.tracings.{TracingIds, TracingType} import mail.{MailchimpClient, MailchimpTag} import models.analytics.{AnalyticsService, CreateAnnotationEvent, OpenAnnotationEvent} @@ -26,7 +27,6 @@ import models.user.time._ import models.user.{User, UserDAO, UserService} import net.liftweb.common.Box import play.api.i18n.{Messages, MessagesProvider} -import play.api.libs.json.Json.WithDefaultValues import play.api.libs.json._ import play.api.mvc.{Action, AnyContent, PlayBodyParsers} import security.{URLSharing, UserAwareRequestLogging, WkEnv} diff --git a/app/controllers/LegacyApiController.scala b/app/controllers/LegacyApiController.scala index 7a3c535ca88..2946ec1ddfa 100644 --- a/app/controllers/LegacyApiController.scala +++ b/app/controllers/LegacyApiController.scala @@ -5,6 +5,7 @@ import play.silhouette.api.actions.SecuredRequest import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.models.VoxelSize import com.scalableminds.webknossos.datastore.models.annotation.{AnnotationLayer, AnnotationLayerType} +import com.scalableminds.webknossos.tracingstore.annotation.AnnotationLayerParameters import com.scalableminds.webknossos.tracingstore.tracings.volume.ResolutionRestrictions import models.dataset.DatasetService import models.organization.OrganizationDAO diff --git a/test/backend/Dummies.scala b/test/backend/Dummies.scala index 8c982c11b99..eb108f94210 100644 --- a/test/backend/Dummies.scala +++ b/test/backend/Dummies.scala @@ -53,6 +53,8 @@ object Dummies { Some(true)) val treeGroup2: TreeGroup = TreeGroup("Axon 2", 2, Seq.empty, Some(true)) + val tracingId: String = "dummyTracingId" + val skeletonTracing: SkeletonTracing = SkeletonTracing( "dummy_dataset", Seq(tree1, tree2), diff --git a/test/backend/SkeletonUpdateActionsUnitTestSuite.scala b/test/backend/SkeletonUpdateActionsUnitTestSuite.scala index 4799ecc342d..3905460e318 100644 --- a/test/backend/SkeletonUpdateActionsUnitTestSuite.scala +++ b/test/backend/SkeletonUpdateActionsUnitTestSuite.scala @@ -2,13 +2,12 @@ package backend import com.scalableminds.util.geometry.{Vec3Int, Vec3Double} import com.scalableminds.webknossos.datastore.SkeletonTracing._ -import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating._ import org.scalatestplus.play._ class SkeletonUpdateActionsUnitTestSuite extends PlaySpec { - private def applyUpdateAction(action: UpdateAction.SkeletonUpdateAction): SkeletonTracing = + private def applyUpdateAction(action: SkeletonUpdateAction): SkeletonTracing = action.applyOn(Dummies.skeletonTracing) def listConsistsOfLists[T](joinedList: Seq[T], sublist1: Seq[T], sublist2: Seq[T]): Boolean = @@ -29,7 +28,8 @@ class SkeletonUpdateActionsUnitTestSuite extends PlaySpec { comments = List[UpdateActionComment](), groupId = None, isVisible = Option(true), - edgesAreVisible = Option(true) + edgesAreVisible = Option(true), + actionTracingId = Dummies.tracingId ) val result = applyUpdateAction(createTreeAction) @@ -46,7 +46,7 @@ class SkeletonUpdateActionsUnitTestSuite extends PlaySpec { "DeleteTreeSkeletonAction" should { "delete the specified tree" in { - val deleteTreeAction = new DeleteTreeSkeletonAction(id = 1) + val deleteTreeAction = new DeleteTreeSkeletonAction(id = 1, actionTracingId = Dummies.tracingId) val result = applyUpdateAction(deleteTreeAction) assert(result.trees.length == Dummies.skeletonTracing.trees.length - 1) @@ -66,7 +66,8 @@ class SkeletonUpdateActionsUnitTestSuite extends PlaySpec { name = "updated tree", branchPoints = List(UpdateActionBranchPoint(0, Dummies.timestamp)), comments = List[UpdateActionComment](), - groupId = None + groupId = None, + actionTracingId = Dummies.tracingId ) val result = applyUpdateAction(updateTreeAction) @@ -81,7 +82,7 @@ class SkeletonUpdateActionsUnitTestSuite extends PlaySpec { "MergeTreeSkeletonAction" should { "merge the specified trees" in { - val mergeTreeAction = new MergeTreeSkeletonAction(sourceId = 1, targetId = 2) + val mergeTreeAction = new MergeTreeSkeletonAction(sourceId = 1, targetId = 2, actionTracingId = Dummies.tracingId) val sourceTree = Dummies.tree1 val targetTree = Dummies.tree2 val result = applyUpdateAction(mergeTreeAction) @@ -102,7 +103,10 @@ class SkeletonUpdateActionsUnitTestSuite extends PlaySpec { "MoveTreeComponentSkeletonAction" should { "move the specified (separate) nodes" in { val moveTreeComponentSkeletonAction = - new MoveTreeComponentSkeletonAction(Dummies.comp1Nodes.map(_.id).toList, sourceId = 3, targetId = 4) + new MoveTreeComponentSkeletonAction(Dummies.comp1Nodes.map(_.id).toList, + sourceId = 3, + targetId = 4, + actionTracingId = Dummies.tracingId) val result = moveTreeComponentSkeletonAction.applyOn(Dummies.componentSkeletonTracing) assert(result.trees.length == Dummies.componentSkeletonTracing.trees.length) @@ -120,7 +124,8 @@ class SkeletonUpdateActionsUnitTestSuite extends PlaySpec { "CreateEdgeSkeletonAction" should { "create a new edge in the right tree" in { - val createEdgeSkeletonAction = new CreateEdgeSkeletonAction(source = 1, target = 7, treeId = 1) + val createEdgeSkeletonAction = + new CreateEdgeSkeletonAction(source = 1, target = 7, treeId = 1, actionTracingId = Dummies.tracingId) val result = applyUpdateAction(createEdgeSkeletonAction) assert(result.trees.length == Dummies.skeletonTracing.trees.length) @@ -133,8 +138,10 @@ class SkeletonUpdateActionsUnitTestSuite extends PlaySpec { "DeleteEdgeSkeletonAction" should { "undo CreateEdgeSkeletonAction" in { - val createEdgeSkeletonAction = new CreateEdgeSkeletonAction(source = 0, target = 7, treeId = 1) - val deleteEdgeSkeletonAction = new DeleteEdgeSkeletonAction(source = 0, target = 7, treeId = 1) + val createEdgeSkeletonAction = + new CreateEdgeSkeletonAction(source = 0, target = 7, treeId = 1, actionTracingId = Dummies.tracingId) + val deleteEdgeSkeletonAction = + new DeleteEdgeSkeletonAction(source = 0, target = 7, treeId = 1, actionTracingId = Dummies.tracingId) val result = deleteEdgeSkeletonAction.applyOn(createEdgeSkeletonAction.applyOn(Dummies.skeletonTracing)) assert(result == Dummies.skeletonTracing) } @@ -154,7 +161,8 @@ class SkeletonUpdateActionsUnitTestSuite extends PlaySpec { Option(newNode.interpolation), treeId = 1, Dummies.timestamp, - None + None, + actionTracingId = Dummies.tracingId ) val result = applyUpdateAction(createNodeSkeletonAction) assert(result.trees.length == Dummies.skeletonTracing.trees.length) @@ -178,7 +186,8 @@ class SkeletonUpdateActionsUnitTestSuite extends PlaySpec { Option(newNode.bitDepth), Option(newNode.interpolation), treeId = 1, - Dummies.timestamp + Dummies.timestamp, + actionTracingId = Dummies.tracingId ) val result = applyUpdateAction(updateNodeSkeletonAction) assert(result.trees.length == Dummies.skeletonTracing.trees.length) @@ -203,9 +212,11 @@ class SkeletonUpdateActionsUnitTestSuite extends PlaySpec { Option(newNode.interpolation), treeId = 1, Dummies.timestamp, - None + None, + actionTracingId = Dummies.tracingId ) - val deleteNodeSkeletonAction = new DeleteNodeSkeletonAction(newNode.id, treeId = 1) + val deleteNodeSkeletonAction = + new DeleteNodeSkeletonAction(newNode.id, treeId = 1, actionTracingId = Dummies.tracingId) val result = deleteNodeSkeletonAction.applyOn(createNodeSkeletonAction.applyOn(Dummies.skeletonTracing)) assert(result == Dummies.skeletonTracing) } @@ -215,7 +226,8 @@ class SkeletonUpdateActionsUnitTestSuite extends PlaySpec { "update a top level tree group" in { val updatedName = "Axon 2 updated" val updateTreeGroupsSkeletonAction = new UpdateTreeGroupsSkeletonAction( - List(UpdateActionTreeGroup(updatedName, 2, Some(true), List())) + List(UpdateActionTreeGroup(updatedName, 2, Some(true), List())), + actionTracingId = Dummies.tracingId ) val result = applyUpdateAction(updateTreeGroupsSkeletonAction) assert(result.trees == Dummies.skeletonTracing.trees) @@ -230,7 +242,8 @@ class SkeletonUpdateActionsUnitTestSuite extends PlaySpec { UpdateActionTreeGroup(updatedNameTop, 1, Some(true), - List(UpdateActionTreeGroup(updatedNameNested, 3, Some(false), List())))) + List(UpdateActionTreeGroup(updatedNameNested, 3, Some(false), List())))), + actionTracingId = Dummies.tracingId ) val result = applyUpdateAction(updateTreeGroupsSkeletonAction) assert(result.trees == Dummies.skeletonTracing.trees) @@ -253,7 +266,8 @@ class SkeletonUpdateActionsUnitTestSuite extends PlaySpec { editPosition, editRotation, zoomLevel, - userBoundingBox + userBoundingBox, + actionTracingId = Dummies.tracingId ) val result = applyUpdateAction(updateTreeGroupsSkeletonAction) assert(result.trees == Dummies.skeletonTracing.trees) diff --git a/test/backend/VolumeUpdateActionsUnitTestSuite.scala b/test/backend/VolumeUpdateActionsUnitTestSuite.scala index 91459fc614b..35dd3f9b0b4 100644 --- a/test/backend/VolumeUpdateActionsUnitTestSuite.scala +++ b/test/backend/VolumeUpdateActionsUnitTestSuite.scala @@ -3,8 +3,8 @@ package backend import com.scalableminds.util.geometry.Vec3Int import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits -import com.scalableminds.webknossos.tracingstore.tracings.UpdateAction import com.scalableminds.webknossos.tracingstore.tracings.volume.{ + ApplyableVolumeUpdateAction, CreateSegmentVolumeAction, DeleteSegmentVolumeAction, UpdateActionSegmentGroup, @@ -15,7 +15,7 @@ import org.scalatestplus.play._ class VolumeUpdateActionsUnitTestSuite extends PlaySpec with ProtoGeometryImplicits { - private def applyUpdateAction(action: UpdateAction.VolumeUpdateAction): VolumeTracing = + private def applyUpdateAction(action: ApplyableVolumeUpdateAction): VolumeTracing = action.applyOn(Dummies.volumeTracing) "CreateSegmentVolumeAction" should { @@ -26,7 +26,8 @@ class VolumeUpdateActionsUnitTestSuite extends PlaySpec with ProtoGeometryImplic color = None, name = Some("aSegment"), groupId = Some(1), - creationTime = Some(Dummies.timestampLong) + creationTime = Some(Dummies.timestampLong), + actionTracingId = Dummies.tracingId ) val result = applyUpdateAction(createSegmentAction) @@ -39,7 +40,7 @@ class VolumeUpdateActionsUnitTestSuite extends PlaySpec with ProtoGeometryImplic "DeleteSegmentVolumeAction" should { "delete the specified segment" in { - val deleteSegmentAction = DeleteSegmentVolumeAction(id = 5) + val deleteSegmentAction = DeleteSegmentVolumeAction(id = 5, actionTracingId = Dummies.tracingId) val result = applyUpdateAction(deleteSegmentAction) assert(result.segments.length == Dummies.volumeTracing.segments.length - 1) @@ -58,7 +59,8 @@ class VolumeUpdateActionsUnitTestSuite extends PlaySpec with ProtoGeometryImplic name = Some("aRenamedSegment"), color = None, creationTime = Some(Dummies.timestampLong), - groupId = None + groupId = None, + actionTracingId = Dummies.tracingId ) val result = applyUpdateAction(updateSegmentAction) @@ -76,7 +78,8 @@ class VolumeUpdateActionsUnitTestSuite extends PlaySpec with ProtoGeometryImplic "update a top level segment group" in { val updatedName = "Segment Group 2 updated" val updateSegmentGroupsVolumeAction = new UpdateSegmentGroupsVolumeAction( - List(UpdateActionSegmentGroup(updatedName, 2, isExpanded = Some(true), List())) + List(UpdateActionSegmentGroup(updatedName, 2, isExpanded = Some(true), List())), + actionTracingId = Dummies.tracingId ) val result = applyUpdateAction(updateSegmentGroupsVolumeAction) assert(result.segments == Dummies.volumeTracing.segments) @@ -87,7 +90,13 @@ class VolumeUpdateActionsUnitTestSuite extends PlaySpec with ProtoGeometryImplic val updatedNameTop = "Segment Group 1 updated" val updatedNameNested = "Segment Group 3 updated" val updateSegmentGroupsVolumeAction = new UpdateSegmentGroupsVolumeAction( - List(UpdateActionSegmentGroup(updatedNameTop, 1, isExpanded = Some(true), List(UpdateActionSegmentGroup(updatedNameNested, 3, isExpanded = Some(false), List())))) + List( + UpdateActionSegmentGroup( + updatedNameTop, + 1, + isExpanded = Some(true), + List(UpdateActionSegmentGroup(updatedNameNested, 3, isExpanded = Some(false), List())))), + actionTracingId = Dummies.tracingId ) val result = applyUpdateAction(updateSegmentGroupsVolumeAction) assert(result.segments == Dummies.volumeTracing.segments) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index dcf07d1d5f8..83ca1ac6ece 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -5,7 +5,7 @@ import com.scalableminds.webknossos.datastore.Annotation.{AnnotationLayerProto, import com.scalableminds.webknossos.datastore.EditableMappingInfo.EditableMappingInfo import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing -import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType +import com.scalableminds.webknossos.datastore.models.annotation.{AnnotationLayer, AnnotationLayerType} import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ EditableMappingUpdateAction, EditableMappingUpdater @@ -44,9 +44,11 @@ case class AnnotationWithTracings( def addTracing(a: AddLayerAnnotationUpdateAction): AnnotationWithTracings = AnnotationWithTracings( annotation.copy( - layers = annotation.layers :+ AnnotationLayerProto(a.tracingId, - a.layerName, - `type` = AnnotationLayerType.toProto(a.`type`))), + layers = annotation.layers :+ AnnotationLayerProto( + a.tracingId, + a.layerParameters.name.getOrElse(AnnotationLayer.defaultNameForType(a.layerParameters.typ)), + `type` = AnnotationLayerType.toProto(a.layerParameters.typ) + )), tracingsById, editableMappingsByTracingId ) From 3e9265eadee8996b0f3bcfcee22702d0d6cfde19 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 27 Aug 2024 10:09:43 +0200 Subject: [PATCH 036/150] initialize annotationProto. wip: call with annotationId --- app/models/annotation/AnnotationService.scala | 24 ++- .../WKRemoteTracingStoreClient.scala | 12 +- .../webknossos/datastore/rpc/RPCRequest.scala | 6 + .../annotation/TSAnnotationService.scala | 30 ++-- .../SkeletonTracingController.scala | 8 +- .../controllers/TSAnnotationController.scala | 12 +- .../controllers/TracingController.scala | 139 ++--------------- .../controllers/VolumeTracingController.scala | 145 ++++++++++++------ .../tracings/TracingSelector.scala | 2 +- .../tracings/TracingService.scala | 35 +++-- .../skeleton/SkeletonTracingService.scala | 59 +------ .../volume/VolumeTracingService.scala | 114 +++++++++----- ...alableminds.webknossos.tracingstore.routes | 66 ++++---- 13 files changed, 313 insertions(+), 339 deletions(-) diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index d84364b475a..9401d9a8b5e 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -7,6 +7,7 @@ import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int} import com.scalableminds.util.io.{NamedStream, ZipIO} import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.{BoxImplicits, Fox, FoxImplicits, TextUtils} +import com.scalableminds.webknossos.datastore.Annotation.{AnnotationLayerProto, AnnotationProto} import com.scalableminds.webknossos.datastore.SkeletonTracing._ import com.scalableminds.webknossos.datastore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings} import com.scalableminds.webknossos.datastore.geometry.{ @@ -205,6 +206,7 @@ class AnnotationService @Inject()( newAnnotationLayers <- createTracingsForExplorational( dataset, dataSource, + annotation._id, List(annotationLayerParameters), organizationName, annotation.annotationLayers) ?~> "annotation.createTracings.failed" @@ -218,6 +220,7 @@ class AnnotationService @Inject()( private def createTracingsForExplorational(dataset: Dataset, dataSource: DataSource, + annotationId: ObjectId, allAnnotationLayerParameters: List[AnnotationLayerParameters], datasetOrganizationName: String, existingAnnotationLayers: List[AnnotationLayer] = List())( @@ -351,6 +354,22 @@ class AnnotationService @Inject()( ) } + def createAndSaveAnnotationProto(annotationId: ObjectId, annotationLayers: List[AnnotationLayer]): Fox[Unit] = { + val layersProto = annotationLayers.map { l => + AnnotationLayerProto( + l.tracingId, + l.name, + AnnotationLayerType.toProto(l.typ) + ) + } + // todo pass right name, description here + val annotationProto = AnnotationProto(name = None, description = None, version = 0L, layers = layersProto) + for { + tracingStoreClient <- tracingStoreService.clientFor(dataset) + _ <- tracingStoreClient.saveAnnotationProto(annotationId, annotationProto) + } yield () + } + for { /* Note that the tracings have redundant properties, with a precedence logic selecting a layer @@ -366,6 +385,7 @@ class AnnotationService @Inject()( precedenceProperties = oldPrecedenceLayer.map(extractPrecedenceProperties) newAnnotationLayers <- Fox.serialCombined(allAnnotationLayerParameters)(p => createAndSaveAnnotationLayer(p, precedenceProperties, dataStore)) + _ <- createAndSaveAnnotationProto(annotationId, newAnnotationLayers) } yield newAnnotationLayers } @@ -393,13 +413,15 @@ class AnnotationService @Inject()( dataSource <- datasetService.dataSourceFor(dataset) datasetOrganization <- organizationDAO.findOne(dataset._organization) usableDataSource <- dataSource.toUsable ?~> Messages("dataset.notImported", dataSource.id.name) + newAnnotationId = ObjectId.generate annotationLayers <- createTracingsForExplorational( dataset, usableDataSource, + newAnnotationId, annotationLayerParameters, datasetOrganization.name) ?~> "annotation.createTracings.failed" teamId <- selectSuitableTeam(user, dataset) ?~> "annotation.create.forbidden" - annotation = Annotation(ObjectId.generate, datasetId, None, teamId, user._id, annotationLayers) + annotation = Annotation(newAnnotationId, datasetId, None, teamId, user._id, annotationLayers) _ <- annotationDAO.insertOne(annotation) } yield annotation diff --git a/app/models/annotation/WKRemoteTracingStoreClient.scala b/app/models/annotation/WKRemoteTracingStoreClient.scala index 95f2c054f02..442a0b1a2ab 100644 --- a/app/models/annotation/WKRemoteTracingStoreClient.scala +++ b/app/models/annotation/WKRemoteTracingStoreClient.scala @@ -6,6 +6,7 @@ import com.scalableminds.util.io.ZipIO import com.scalableminds.util.tools.Fox import com.scalableminds.util.tools.Fox.bool2Fox import com.scalableminds.util.tools.JsonHelper.{boxFormat, optionFormat} +import com.scalableminds.webknossos.datastore.Annotation.AnnotationProto import com.scalableminds.webknossos.datastore.SkeletonTracing.{SkeletonTracing, SkeletonTracings} import com.scalableminds.webknossos.datastore.VolumeTracing.{VolumeTracing, VolumeTracings} import com.scalableminds.webknossos.datastore.models.VoxelSize @@ -23,6 +24,7 @@ import com.typesafe.scalalogging.LazyLogging import controllers.RpcTokenHolder import models.dataset.Dataset import net.liftweb.common.Box +import utils.ObjectId import scala.concurrent.ExecutionContext @@ -33,7 +35,7 @@ class WKRemoteTracingStoreClient( tracingDataSourceTemporaryStore: TracingDataSourceTemporaryStore)(implicit ec: ExecutionContext) extends LazyLogging { - def baseInfo = s" Dataset: ${dataset.name} Tracingstore: ${tracingStore.url}" + private def baseInfo = s" Dataset: ${dataset.name} Tracingstore: ${tracingStore.url}" def getSkeletonTracing(annotationLayer: AnnotationLayer, version: Option[Long]): Fox[FetchedAnnotationLayer] = { logger.debug("Called to get SkeletonTracing." + baseInfo) @@ -80,6 +82,14 @@ class WKRemoteTracingStoreClient( .postProtoWithJsonResponse[SkeletonTracings, List[Box[Option[String]]]](tracings) } + def saveAnnotationProto(annotationId: ObjectId, annotationProto: AnnotationProto): Fox[Unit] = { + logger.debug("Called to save AnnotationProto." + baseInfo) + rpc(s"${tracingStore.url}/annotations/save") + .addQueryString("token" -> RpcTokenHolder.webknossosToken) + .addQueryString("annotationId" -> annotationId.toString) + .postProto[AnnotationProto](annotationProto) + } + def duplicateSkeletonTracing(skeletonTracingId: String, versionString: Option[String] = None, isFromTask: Boolean = false, diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala index e1b36d40e42..1d99f8aab03 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala @@ -167,6 +167,12 @@ class RPCRequest(val id: Int, val url: String, wsClient: WSClient)(implicit ec: performRequest } + def postProto[T <: GeneratedMessage](body: T): Fox[Unit] = { + request = + request.addHttpHeaders(HeaderNames.CONTENT_TYPE -> protobufMimeType).withBody(body.toByteArray).withMethod("POST") + performRequest.map(_ => ()) + } + def postProtoWithJsonResponse[T <: GeneratedMessage, J: Reads](body: T): Fox[J] = { request = request.addHttpHeaders(HeaderNames.CONTENT_TYPE -> protobufMimeType).withBody(body.toByteArray).withMethod("POST") diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index c972d51358f..dc333ee665f 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -61,7 +61,8 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl preprocessActionsForStorage(updateActionGroup)) bucketMutatingActions = findBucketMutatingActions(updateActionGroup) _ <- Fox.runIf(bucketMutatingActions.nonEmpty)( - volumeTracingService.applyBucketMutatingActions(bucketMutatingActions, updateActionGroup.version, userToken)) + volumeTracingService + .applyBucketMutatingActions(annotationId, bucketMutatingActions, updateActionGroup.version, userToken)) } yield () private def findBucketMutatingActions(updateActionGroup: UpdateActionGroup): List[BucketMutatingVolumeUpdateAction] = @@ -140,25 +141,34 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl } yield Json.toJson(updateActionGroupsJs) } - def get(annotationId: String, version: Option[Long], applyUpdates: Boolean, userToken: Option[String])( + def get(annotationId: String, version: Option[Long], userToken: Option[String])( implicit ec: ExecutionContext): Fox[AnnotationProto] = + for { + withTracings <- getWithTracings(annotationId, version, List.empty, userToken) + } yield withTracings.annotation + + def getWithTracings(annotationId: String, + version: Option[Long], + requestedTracingIds: List[String], + userToken: Option[String])(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { annotationWithVersion <- tracingDataStore.annotations.get(annotationId, version)(fromProtoBytes[AnnotationProto]) annotation = annotationWithVersion.value - updated <- if (applyUpdates) applyPendingUpdates(annotation, annotationId, version, userToken) - else Fox.successful(annotation) + updated <- applyPendingUpdates(annotation, annotationId, version, requestedTracingIds, userToken) } yield updated - private def applyPendingUpdates(annotation: AnnotationProto, - annotationId: String, - targetVersionOpt: Option[Long], - userToken: Option[String])(implicit ec: ExecutionContext): Fox[AnnotationProto] = + private def applyPendingUpdates( + annotation: AnnotationProto, + annotationId: String, + targetVersionOpt: Option[Long], + requestedTracingIds: List[String], + userToken: Option[String])(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { targetVersion <- determineTargetVersion(annotation, annotationId, targetVersionOpt) updates <- findPendingUpdates(annotationId, annotation.version, targetVersion) - annotationWithTracings <- findTracingsForUpdates(annotation, updates) + annotationWithTracings <- findTracingsForUpdates(annotation, updates) // TODO pass requested tracing ids updated <- applyUpdates(annotationWithTracings, annotationId, updates, targetVersion, userToken) - } yield updated.annotation + } yield updated private def findTracingsForUpdates(annotation: AnnotationProto, updates: List[UpdateAction])( implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala index b2d0640788f..6bb280fe104 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala @@ -50,6 +50,7 @@ class SkeletonTracingController @Inject()(val tracingService: SkeletonTracingSer } def duplicate(token: Option[String], + annotationId: String, tracingId: String, version: Option[Long], fromTask: Option[Boolean], @@ -60,7 +61,12 @@ class SkeletonTracingController @Inject()(val tracingService: SkeletonTracingSer log() { accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId, version, applyUpdates = true) ?~> Messages("tracing.notFound") + tracing <- tracingService.find(annotationId, + tracingId, + version, + applyUpdates = true, + userToken = urlOrHeaderToken(token, request)) ?~> Messages( + "tracing.notFound") editPositionParsed <- Fox.runOptional(editPosition)(Vec3Int.fromUriLiteral) editRotationParsed <- Fox.runOptional(editRotation)(Vec3Double.fromUriLiteral) boundingBoxParsed <- Fox.runOptional(boundingBox)(BoundingBox.fromLiteral) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index 6739b3697d0..b6873f0fa73 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -25,12 +25,13 @@ class TSAnnotationController @Inject()( extends Controller with KeyValueStoreImplicits { - def initialize(token: Option[String], annotationId: String): Action[AnyContent] = - Action.async { implicit request => + def save(token: Option[String], annotationId: String): Action[AnnotationProto] = + Action.async(validateProto[AnnotationProto]) { implicit request => log() { accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { for { - _ <- tracingDataStore.annotations.put(annotationId, 0L, AnnotationProto(version = 0L)) + // TODO assert id does not already exist + _ <- tracingDataStore.annotations.put(annotationId, 0L, request.body) } yield Ok } } @@ -84,10 +85,7 @@ class TSAnnotationController @Inject()( accessTokenService.validateAccess(UserAccessRequest.readAnnotation(annotationId), urlOrHeaderToken(token, request)) { for { - annotationProto <- annotationService.get(annotationId, - version, - applyUpdates = false, - urlOrHeaderToken(token, request)) + annotationProto <- annotationService.get(annotationId, version, urlOrHeaderToken(token, request)) } yield Ok(annotationProto.toByteArray).as(protobufMimeType) } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala index bad8308b096..580593fccd8 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala @@ -72,18 +72,21 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C } } - def get(token: Option[String], tracingId: String, version: Option[Long]): Action[AnyContent] = Action.async { - implicit request => + def get(token: Option[String], annotationId: String, tracingId: String, version: Option[Long]): Action[AnyContent] = + Action.async { implicit request => log() { accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId, version, applyUpdates = true) ?~> Messages("tracing.notFound") - } yield { - Ok(tracing.toByteArray).as(protobufMimeType) - } + tracing <- tracingService.find(annotationId, + tracingId, + version, + applyUpdates = true, + userToken = urlOrHeaderToken(token, request)) ?~> Messages( + "tracing.notFound") + } yield Ok(tracing.toByteArray).as(protobufMimeType) } } - } + } def getMultiple(token: Option[String]): Action[List[Option[TracingSelector]]] = Action.async(validateJson[List[Option[TracingSelector]]]) { implicit request => @@ -98,127 +101,13 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C } } - def newestVersion(token: Option[String], tracingId: String): Action[AnyContent] = Action.async { implicit request => - log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), token) { - for { - newestVersion <- tracingService.currentVersion(tracingId) ?~> "annotation.getNewestVersion.failed" - } yield { - JsonOk(Json.obj("version" -> newestVersion)) - } - } - } - } - - def update(token: Option[String], tracingId: String): Action[List[UpdateActionGroup]] = - Action.async(validateJson[List[UpdateActionGroup]]) { implicit request => + def newestVersion(token: Option[String], annotationId: String, tracingId: String): Action[AnyContent] = Action.async { + implicit request => log() { - logTime(slackNotificationService.noticeSlowRequest) { - accessTokenService.validateAccess(UserAccessRequest.writeTracing(tracingId), urlOrHeaderToken(token, request)) { - val updateGroups = request.body - if (updateGroups.forall(_.transactionGroupCount == 1)) { - commitUpdates(tracingId, updateGroups, urlOrHeaderToken(token, request)).map(_ => Ok) - } else { - updateGroups - .foldLeft(tracingService.currentVersion(tracingId)) { (currentCommittedVersionFox, updateGroup) => - handleUpdateGroupForTransaction(tracingId, - currentCommittedVersionFox, - updateGroup, - urlOrHeaderToken(token, request)) - } - .map(_ => Ok) - } - } - } - } - } - - private val transactionGroupExpiry: FiniteDuration = 24 hours - - private def handleUpdateGroupForTransaction(tracingId: String, - previousVersionFox: Fox[Long], - updateGroup: UpdateActionGroup, - userToken: Option[String]): Fox[Long] = - for { - previousCommittedVersion: Long <- previousVersionFox - result <- if (previousCommittedVersion + 1 == updateGroup.version) { - if (updateGroup.transactionGroupCount == updateGroup.transactionGroupIndex + 1) { - // Received the last group of this transaction - commitWithPending(tracingId, updateGroup, userToken) - } else { - tracingService - .saveUncommitted(tracingId, - updateGroup.transactionId, - updateGroup.transactionGroupIndex, - updateGroup.version, - updateGroup, - transactionGroupExpiry) - .flatMap( - _ => - tracingService.saveToHandledGroupIdStore(tracingId, - updateGroup.transactionId, - updateGroup.version, - updateGroup.transactionGroupIndex)) - .map(_ => previousCommittedVersion) // no updates have been committed, do not yield version increase + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), token) { + Fox.successful(JsonOk(Json.obj("version" -> 0L))) // TODO remove in favor of annotation-wide } - } else { - failUnlessAlreadyHandled(updateGroup, tracingId, previousCommittedVersion) } - } yield result - - // For an update group (that is the last of a transaction), fetch all previous uncommitted for the same transaction - // and commit them all. - private def commitWithPending(tracingId: String, - updateGroup: UpdateActionGroup, - userToken: Option[String]): Fox[Long] = - for { - previousActionGroupsToCommit <- tracingService.getAllUncommittedFor(tracingId, updateGroup.transactionId) - _ <- bool2Fox( - previousActionGroupsToCommit - .exists(_.transactionGroupIndex == 0) || updateGroup.transactionGroupCount == 1) ?~> s"Trying to commit a transaction without a group that has transactionGroupIndex 0." - concatenatedGroup = concatenateUpdateGroupsOfTransaction(previousActionGroupsToCommit, updateGroup) - commitResult <- commitUpdates(tracingId, List(concatenatedGroup), userToken) - _ <- tracingService.removeAllUncommittedFor(tracingId, updateGroup.transactionId) - } yield commitResult - - private def concatenateUpdateGroupsOfTransaction(previousActionGroups: List[UpdateActionGroup], - lastActionGroup: UpdateActionGroup): UpdateActionGroup = - if (previousActionGroups.isEmpty) lastActionGroup - else { - val allActionGroups = previousActionGroups :+ lastActionGroup - UpdateActionGroup( - version = lastActionGroup.version, - timestamp = lastActionGroup.timestamp, - authorId = lastActionGroup.authorId, - actions = allActionGroups.flatMap(_.actions), - stats = lastActionGroup.stats, // the latest stats do count - info = lastActionGroup.info, // frontend sets this identically for all groups of transaction - transactionId = f"${lastActionGroup.transactionId}-concatenated", - transactionGroupCount = 1, - transactionGroupIndex = 0, - ) - } - - // Perform version check and commit the passed updates - private def commitUpdates(tracingId: String, - updateGroups: List[UpdateActionGroup], - userToken: Option[String]): Fox[Long] = ??? - - /* If this update group has already been “handled” (successfully saved as either committed or uncommitted), - * ignore it silently. This is in case the frontend sends a retry if it believes a save to be unsuccessful - * despite the backend receiving it just fine. - */ - private def failUnlessAlreadyHandled(updateGroup: UpdateActionGroup, - tracingId: String, - previousVersion: Long): Fox[Long] = { - val errorMessage = s"Incorrect version. Expected: ${previousVersion + 1}; Got: ${updateGroup.version}" - for { - _ <- Fox.assertTrue( - tracingService.handledGroupIdStoreContains(tracingId, - updateGroup.transactionId, - updateGroup.version, - updateGroup.transactionGroupIndex)) ?~> errorMessage ~> CONFLICT - } yield updateGroup.version } def mergedFromIds(token: Option[String], persist: Boolean): Action[List[Option[TracingSelector]]] = diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 4f24fcdccb0..6b5671eddaf 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -90,6 +90,7 @@ class VolumeTracingController @Inject()( tracings.tracings.toList.map(_.tracing) def initialData(token: Option[String], + annotationId: String, tracingId: String, minResolution: Option[Int], maxResolution: Option[Int]): Action[AnyContent] = @@ -99,10 +100,11 @@ class VolumeTracingController @Inject()( accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { for { initialData <- request.body.asRaw.map(_.asFile) ?~> Messages("zipFile.notFound") - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( + "tracing.notFound") resolutionRestrictions = ResolutionRestrictions(minResolution, maxResolution) resolutions <- tracingService - .initializeWithData(tracingId, tracing, initialData, resolutionRestrictions, token) + .initializeWithData(annotationId, tracingId, tracing, initialData, resolutionRestrictions, token) .toFox _ <- tracingService.updateResolutionList(tracingId, tracing, resolutions) } yield Ok(Json.toJson(tracingId)) @@ -130,23 +132,27 @@ class VolumeTracingController @Inject()( } } - def initialDataMultiple(token: Option[String], tracingId: String): Action[AnyContent] = Action.async { - implicit request => + def initialDataMultiple(token: Option[String], annotationId: String, tracingId: String): Action[AnyContent] = + Action.async { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { for { initialData <- request.body.asRaw.map(_.asFile) ?~> Messages("zipFile.notFound") - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") - resolutions <- tracingService.initializeWithDataMultiple(tracingId, tracing, initialData, token).toFox + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( + "tracing.notFound") + resolutions <- tracingService + .initializeWithDataMultiple(annotationId, tracingId, tracing, initialData, token) + .toFox _ <- tracingService.updateResolutionList(tracingId, tracing, resolutions) } yield Ok(Json.toJson(tracingId)) } } } - } + } def allDataZip(token: Option[String], + annotationId: String, tracingId: String, volumeDataZipFormat: String, version: Option[Long], @@ -156,7 +162,11 @@ class VolumeTracingController @Inject()( log() { accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId, version) ?~> Messages("tracing.notFound") + tracing <- tracingService.find(annotationId, + tracingId, + version, + userToken = urlOrHeaderToken(token, request)) ?~> Messages( + "tracing.notFound") volumeDataZipFormatParsed <- VolumeDataZipFormat.fromString(volumeDataZipFormat).toFox voxelSizeFactorParsedOpt <- Fox.runOptional(voxelSizeFactor)(Vec3Double.fromUriLiteral) voxelSizeUnitParsedOpt <- Fox.runOptional(voxelSizeUnit)(LengthUnit.fromString) @@ -173,12 +183,13 @@ class VolumeTracingController @Inject()( } } - def data(token: Option[String], tracingId: String): Action[List[WebknossosDataRequest]] = + def data(token: Option[String], annotationId: String, tracingId: String): Action[List[WebknossosDataRequest]] = Action.async(validateJson[List[WebknossosDataRequest]]) { implicit request => log() { accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( + "tracing.notFound") (data, indices) <- if (tracing.getHasEditableMapping) editableMappingService.volumeData(tracing, tracingId, request.body, urlOrHeaderToken(token, request)) else tracingService.data(tracingId, tracing, request.body) @@ -194,6 +205,7 @@ class VolumeTracingController @Inject()( "[" + indices.mkString(", ") + "]" def duplicate(token: Option[String], + annotationId: String, tracingId: String, fromTask: Option[Boolean], minResolution: Option[Int], @@ -207,7 +219,8 @@ class VolumeTracingController @Inject()( val userToken = urlOrHeaderToken(token, request) accessTokenService.validateAccess(UserAccessRequest.webknossos, userToken) { for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") + tracing <- tracingService.find(annotationId, tracingId, userToken = userToken) ?~> Messages( + "tracing.notFound") _ = logger.info(s"Duplicating volume tracing $tracingId...") datasetBoundingBox = request.body.asJson.flatMap(_.validateOpt[BoundingBox].asOpt.flatten) resolutionRestrictions = ResolutionRestrictions(minResolution, maxResolution) @@ -219,6 +232,7 @@ class VolumeTracingController @Inject()( newEditableMappingId <- Fox.runIf(tracing.getHasEditableMapping)( editableMappingService.duplicate(tracing.mappingName, version = None, remoteFallbackLayerOpt, userToken)) (newId, newTracing) <- tracingService.duplicate( + annotationId, tracingId, tracing, fromTask.getOrElse(false), @@ -237,15 +251,19 @@ class VolumeTracingController @Inject()( } } - def importVolumeData(token: Option[String], tracingId: String): Action[MultipartFormData[TemporaryFile]] = + def importVolumeData(token: Option[String], + annotationId: String, + tracingId: String): Action[MultipartFormData[TemporaryFile]] = Action.async(parse.multipartFormData) { implicit request => log() { accessTokenService.validateAccess(UserAccessRequest.writeTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( + "tracing.notFound") currentVersion <- request.body.dataParts("currentVersion").headOption.flatMap(_.toIntOpt).toFox zipFile <- request.body.files.headOption.map(f => new File(f.ref.path.toString)).toFox - largestSegmentId <- tracingService.importVolumeData(tracingId, + largestSegmentId <- tracingService.importVolumeData(annotationId, + tracingId, tracing, zipFile, currentVersion, @@ -255,17 +273,22 @@ class VolumeTracingController @Inject()( } } - def addSegmentIndex(token: Option[String], tracingId: String, dryRun: Boolean): Action[AnyContent] = + def addSegmentIndex(token: Option[String], + annotationId: String, + tracingId: String, + dryRun: Boolean): Action[AnyContent] = Action.async { implicit request => log() { accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) ?~> "tracing.notFound" + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( + "tracing.notFound") currentVersion <- tracingService.currentVersion(tracingId) before = Instant.now canAddSegmentIndex <- tracingService.checkIfSegmentIndexMayBeAdded(tracingId, tracing, token) processedBucketCountOpt <- Fox.runIf(canAddSegmentIndex)( - tracingService.addSegmentIndex(tracingId, + tracingService.addSegmentIndex(annotationId, + tracingId, tracing, currentVersion, urlOrHeaderToken(token, request), @@ -296,17 +319,20 @@ class VolumeTracingController @Inject()( } } - def requestAdHocMesh(token: Option[String], tracingId: String): Action[WebknossosAdHocMeshRequest] = + def requestAdHocMesh(token: Option[String], + annotationId: String, + tracingId: String): Action[WebknossosAdHocMeshRequest] = Action.async(validateJson[WebknossosAdHocMeshRequest]) { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { // The client expects the ad-hoc mesh as a flat float-array. Three consecutive floats form a 3D point, three // consecutive 3D points (i.e., nine floats) form a triangle. // There are no shared vertices between triangles. - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( + "tracing.notFound") (vertices, neighbors) <- if (tracing.getHasEditableMapping) editableMappingService.createAdHocMesh(tracing, tracingId, request.body, urlOrHeaderToken(token, request)) - else tracingService.createAdHocMesh(tracingId, request.body, urlOrHeaderToken(token, request)) + else tracingService.createAdHocMesh(annotationId, tracingId, request.body, urlOrHeaderToken(token, request)) } yield { // We need four bytes for each float val responseBuffer = ByteBuffer.allocate(vertices.length * 4).order(ByteOrder.LITTLE_ENDIAN) @@ -331,21 +357,25 @@ class VolumeTracingController @Inject()( private def formatNeighborList(neighbors: List[Int]): String = "[" + neighbors.mkString(", ") + "]" - def findData(token: Option[String], tracingId: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { - for { - positionOpt <- tracingService.findData(tracingId) - } yield { - Ok(Json.obj("position" -> positionOpt, "resolution" -> positionOpt.map(_ => Vec3Int.ones))) + def findData(token: Option[String], annotationId: String, tracingId: String): Action[AnyContent] = Action.async { + implicit request => + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + for { + positionOpt <- tracingService.findData(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) + } yield { + Ok(Json.obj("position" -> positionOpt, "resolution" -> positionOpt.map(_ => Vec3Int.ones))) + } } - } } - def agglomerateSkeleton(token: Option[String], tracingId: String, agglomerateId: Long): Action[AnyContent] = + def agglomerateSkeleton(token: Option[String], + annotationId: String, + tracingId: String, + agglomerateId: Long): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) _ <- bool2Fox(tracing.getHasEditableMapping) ?~> "Cannot query agglomerate skeleton for volume annotation" mappingName <- tracing.mappingName ?~> "annotation.agglomerateSkeleton.noMappingSet" remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) @@ -358,12 +388,12 @@ class VolumeTracingController @Inject()( } } - def makeMappingEditable(token: Option[String], tracingId: String): Action[AnyContent] = + def makeMappingEditable(token: Option[String], annotationId: String, tracingId: String): Action[AnyContent] = Action.async { implicit request => log() { accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) tracingMappingName <- tracing.mappingName ?~> "annotation.noMappingSet" _ <- assertMappingIsNotLocked(tracing) _ <- bool2Fox(tracingService.volumeBucketsAreEmpty(tracingId)) ?~> "annotation.volumeBucketsNotEmpty" @@ -400,12 +430,12 @@ class VolumeTracingController @Inject()( private def assertMappingIsNotLocked(volumeTracing: VolumeTracing): Fox[Unit] = bool2Fox(!volumeTracing.mappingIsLocked.getOrElse(false)) ?~> "annotation.mappingIsLocked" - def agglomerateGraphMinCut(token: Option[String], tracingId: String): Action[MinCutParameters] = + def agglomerateGraphMinCut(token: Option[String], annotationId: String, tracingId: String): Action[MinCutParameters] = Action.async(validateJson[MinCutParameters]) { implicit request => log() { accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) _ <- bool2Fox(tracing.getHasEditableMapping) ?~> "Mapping is not editable" remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) edges <- editableMappingService.agglomerateGraphMinCut(request.body, remoteFallbackLayer, token) @@ -414,12 +444,14 @@ class VolumeTracingController @Inject()( } } - def agglomerateGraphNeighbors(token: Option[String], tracingId: String): Action[NeighborsParameters] = + def agglomerateGraphNeighbors(token: Option[String], + annotationId: String, + tracingId: String): Action[NeighborsParameters] = Action.async(validateJson[NeighborsParameters]) { implicit request => log() { accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) _ <- bool2Fox(tracing.getHasEditableMapping) ?~> "Mapping is not editable" remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) (segmentId, edges) <- editableMappingService.agglomerateGraphNeighbors(request.body, @@ -430,11 +462,13 @@ class VolumeTracingController @Inject()( } } - def updateEditableMapping(token: Option[String], tracingId: String): Action[List[UpdateActionGroup]] = + def updateEditableMapping(token: Option[String], + annotationId: String, + tracingId: String): Action[List[UpdateActionGroup]] = Action.async(validateJson[List[UpdateActionGroup]]) { implicit request => accessTokenService.validateAccess(UserAccessRequest.writeTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) mappingName <- tracing.mappingName.toFox _ <- bool2Fox(tracing.getHasEditableMapping) ?~> "Mapping is not editable" currentVersion <- editableMappingService.getClosestMaterializableVersionOrZero(mappingName, None) @@ -460,12 +494,15 @@ class VolumeTracingController @Inject()( } } - def editableMappingInfo(token: Option[String], tracingId: String, version: Option[Long]): Action[AnyContent] = + def editableMappingInfo(token: Option[String], + annotationId: String, + tracingId: String, + version: Option[Long]): Action[AnyContent] = Action.async { implicit request => log() { accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) mappingName <- tracing.mappingName.toFox remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) editableMappingInfo <- editableMappingService.getInfo(mappingName, @@ -481,12 +518,14 @@ class VolumeTracingController @Inject()( } } - def editableMappingAgglomerateIdsForSegments(token: Option[String], tracingId: String): Action[ListOfLong] = + def editableMappingAgglomerateIdsForSegments(token: Option[String], + annotationId: String, + tracingId: String): Action[ListOfLong] = Action.async(validateProto[ListOfLong]) { implicit request => log() { accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) editableMappingId <- tracing.mappingName.toFox remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) (editableMappingInfo, editableMappingVersion) <- editableMappingService.getInfoAndActualVersion( @@ -508,13 +547,14 @@ class VolumeTracingController @Inject()( } def editableMappingSegmentIdsForAgglomerate(token: Option[String], + annotationId: String, tracingId: String, agglomerateId: Long): Action[AnyContent] = Action.async { implicit request => log() { accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) mappingName <- tracing.mappingName.toFox remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) agglomerateGraphBox: Box[AgglomerateGraph] <- editableMappingService @@ -534,11 +574,13 @@ class VolumeTracingController @Inject()( } } - def getSegmentVolume(token: Option[String], tracingId: String): Action[SegmentStatisticsParameters] = + def getSegmentVolume(token: Option[String], + annotationId: String, + tracingId: String): Action[SegmentStatisticsParameters] = Action.async(validateJson[SegmentStatisticsParameters]) { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) mappingName <- tracingService.baseMappingName(tracing) segmentVolumes <- Fox.serialCombined(request.body.segmentIds) { segmentId => volumeSegmentStatisticsService.getSegmentVolume(tracingId, @@ -552,11 +594,13 @@ class VolumeTracingController @Inject()( } } - def getSegmentBoundingBox(token: Option[String], tracingId: String): Action[SegmentStatisticsParameters] = + def getSegmentBoundingBox(token: Option[String], + annotationId: String, + tracingId: String): Action[SegmentStatisticsParameters] = Action.async(validateJson[SegmentStatisticsParameters]) { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) mappingName <- tracingService.baseMappingName(tracing) segmentBoundingBoxes: List[BoundingBox] <- Fox.serialCombined(request.body.segmentIds) { segmentId => volumeSegmentStatisticsService.getSegmentBoundingBox(tracingId, @@ -570,12 +614,15 @@ class VolumeTracingController @Inject()( } } - def getSegmentIndex(token: Option[String], tracingId: String, segmentId: Long): Action[GetSegmentIndexParameters] = + def getSegmentIndex(token: Option[String], + annotationId: String, + tracingId: String, + segmentId: Long): Action[GetSegmentIndexParameters] = Action.async(validateJson[GetSegmentIndexParameters]) { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - fallbackLayer <- tracingService.getFallbackLayer(tracingId) - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") + fallbackLayer <- tracingService.getFallbackLayer(annotationId, tracingId, urlOrHeaderToken(token, request)) + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) mappingName <- tracingService.baseMappingName(tracing) _ <- bool2Fox(DataLayer.bucketSize <= request.body.cubeSize) ?~> "cubeSize must be at least one bucket (32³)" bucketPositionsRaw: ListOfVec3IntProto <- volumeSegmentIndexService diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingSelector.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingSelector.scala index 0329c57e34a..14598c9d5aa 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingSelector.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingSelector.scala @@ -2,6 +2,6 @@ package com.scalableminds.webknossos.tracingstore.tracings import play.api.libs.json.{Json, OFormat} -case class TracingSelector(tracingId: String, version: Option[Long] = None) +case class TracingSelector(tracingId: String, version: Option[Long] = None) // TODO must pass annotation id object TracingSelector { implicit val jsonFormat: OFormat[TracingSelector] = Json.format[TracingSelector] } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala index 7eacf130c2a..8bba698ee1b 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala @@ -1,8 +1,13 @@ package com.scalableminds.webknossos.tracingstore.tracings import com.scalableminds.util.tools.{Fox, FoxImplicits, JsonHelper} +import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.tracingstore.TracingStoreRedisStore -import com.scalableminds.webknossos.tracingstore.annotation.UpdateActionGroup +import com.scalableminds.webknossos.tracingstore.annotation.{ + AnnotationWithTracings, + TSAnnotationService, + UpdateActionGroup +} import com.scalableminds.webknossos.tracingstore.tracings.TracingType.TracingType import com.scalableminds.webknossos.tracingstore.tracings.volume.MergedVolumeStats import com.typesafe.scalalogging.LazyLogging @@ -39,6 +44,8 @@ trait TracingService[T <: GeneratedMessage] def tracingMigrationService: TracingMigrationService[T] + def annotationService: TSAnnotationService + def dummyTracing: T val handledGroupIdStore: TracingStoreRedisStore @@ -110,27 +117,21 @@ trait TracingService[T <: GeneratedMessage] def applyPendingUpdates(tracing: T, tracingId: String, targetVersion: Option[Long]): Fox[T] = Fox.successful(tracing) - def find(tracingId: String, + protected def takeTracing(annotation: AnnotationWithTracings, tracingId: String): Box[T] + + def find(annotationId: String, + tracingId: String, version: Option[Long] = None, useCache: Boolean = true, - applyUpdates: Boolean = false): Fox[T] = + applyUpdates: Boolean = false, + userToken: Option[String]): Fox[T] = if (tracingId == TracingIds.dummyTracingId) Fox.successful(dummyTracing) else { - val tracingFox = tracingStore.get(tracingId, version)(fromProtoBytes[T]).map(_.value) - tracingFox.flatMap { tracing => - val updatedTracing = if (applyUpdates) { - applyPendingUpdates(tracing, tracingId, version) - } else { - Fox.successful(tracing) - } - migrateTracing(updatedTracing, tracingId) - }.orElse { - if (useCache) - temporaryTracingStore.find(tracingId) - else - tracingFox - } + for { + annotation <- annotationService.getWithTracings(annotationId, version, List(tracingId), userToken) // TODO is applyUpdates still needed? + tracing <- takeTracing(annotation, annotationId) + } yield tracing } def findMultiple(selectors: List[Option[TracingSelector]], diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala index a70eb859f7d..15b77bf10f0 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala @@ -8,7 +8,11 @@ import com.scalableminds.webknossos.datastore.geometry.NamedBoundingBoxProto import com.scalableminds.webknossos.datastore.helpers.{ProtoGeometryImplicits, SkeletonTracingDefaults} import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis import com.scalableminds.webknossos.tracingstore.TracingStoreRedisStore -import com.scalableminds.webknossos.tracingstore.annotation.LayerUpdateAction +import com.scalableminds.webknossos.tracingstore.annotation.{ + AnnotationWithTracings, + LayerUpdateAction, + TSAnnotationService +} import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating._ import com.scalableminds.webknossos.tracingstore.tracings.volume.MergedVolumeStats @@ -23,6 +27,7 @@ class SkeletonTracingService @Inject()( val handledGroupIdStore: TracingStoreRedisStore, val temporaryTracingIdStore: TracingStoreRedisStore, val uncommittedUpdatesStore: TracingStoreRedisStore, + val annotationService: TSAnnotationService, val tracingMigrationService: SkeletonTracingMigrationService)(implicit val ec: ExecutionContext) extends TracingService[SkeletonTracing] with KeyValueStoreImplicits @@ -40,22 +45,8 @@ class SkeletonTracingService @Inject()( def currentVersion(tracing: SkeletonTracing): Long = tracing.version - override def applyPendingUpdates(tracing: SkeletonTracing, - tracingId: String, - desiredVersion: Option[Long]): Fox[SkeletonTracing] = { - val existingVersion = tracing.version - findDesiredOrNewestPossibleVersion(tracing, tracingId, desiredVersion).flatMap { newVersion => - if (newVersion > existingVersion) { - for { - pendingUpdates <- findPendingUpdates(tracingId, existingVersion, newVersion) - updatedTracing <- update(tracing, tracingId, pendingUpdates, newVersion) - _ <- save(updatedTracing, Some(tracingId), newVersion) - } yield updatedTracing - } else { - Full(tracing) - } - } - } + protected def takeTracing(annotation: AnnotationWithTracings, tracingId: String): Box[SkeletonTracing] = + annotation.getSkeleton(tracingId) private def findDesiredOrNewestPossibleVersion(tracing: SkeletonTracing, tracingId: String, @@ -76,40 +67,6 @@ class SkeletonTracingService @Inject()( } } - private def findPendingUpdates(tracingId: String, - existingVersion: Long, - desiredVersion: Long): Fox[List[SkeletonUpdateAction]] = ??? - - private def applyUpdateOn(tracing: SkeletonTracing, update: LayerUpdateAction): SkeletonTracing = ??? - - private def update(tracing: SkeletonTracing, - tracingId: String, - updates: List[SkeletonUpdateAction], - newVersion: Long): Fox[SkeletonTracing] = { - def updateIter(tracingFox: Fox[SkeletonTracing], - remainingUpdates: List[SkeletonUpdateAction]): Fox[SkeletonTracing] = - tracingFox.futureBox.flatMap { - case Empty => Fox.empty - case Full(tracing) => - remainingUpdates match { - case List() => Fox.successful(tracing) - case RevertToVersionSkeletonAction(sourceVersion, tracingId, _, _, _) :: tail => - val sourceTracing = find(tracingId, Some(sourceVersion), useCache = false, applyUpdates = true) - updateIter(sourceTracing, tail) - case update :: tail => updateIter(Full(applyUpdateOn(tracing, update)), tail) - } - case _ => tracingFox - } - - updates match { - case List() => Full(tracing) - case _ :: _ => - for { - updated <- updateIter(Some(tracing), updates) - } yield updated.withVersion(newVersion) - } - } - def duplicate(tracing: SkeletonTracing, fromTask: Boolean, editPosition: Option[Vec3Int], diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 95dae6636b7..efeef014b49 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -6,6 +6,7 @@ import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int} import com.scalableminds.util.io.{NamedStream, ZipIO} import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.{Fox, FoxImplicits} +import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.dataformats.wkw.WKWDataFormatHelper import com.scalableminds.webknossos.datastore.geometry.NamedBoundingBoxProto @@ -22,7 +23,11 @@ import com.scalableminds.webknossos.datastore.models.{ WebknossosAdHocMeshRequest } import com.scalableminds.webknossos.datastore.services._ -import com.scalableminds.webknossos.tracingstore.annotation.UpdateActionGroup +import com.scalableminds.webknossos.tracingstore.annotation.{ + AnnotationWithTracings, + TSAnnotationService, + UpdateActionGroup +} import com.scalableminds.webknossos.tracingstore.tracings.TracingType.TracingType import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService @@ -58,6 +63,7 @@ class VolumeTracingService @Inject()( editableMappingService: EditableMappingService, val temporaryTracingIdStore: TracingStoreRedisStore, val remoteDatastoreClient: TSRemoteDatastoreClient, + val annotationService: TSAnnotationService, val remoteWebknossosClient: TSRemoteWebknossosClient, val temporaryFileCreator: TemporaryFileCreator, val tracingMigrationService: VolumeTracingMigrationService, @@ -90,7 +96,8 @@ class VolumeTracingService @Inject()( adHocMeshServiceHolder.tracingStoreAdHocMeshConfig = (binaryDataService, 30 seconds, 1) val adHocMeshService: AdHocMeshService = adHocMeshServiceHolder.tracingStoreAdHocMeshService - private val fallbackLayerCache: AlfuCache[String, Option[RemoteFallbackLayer]] = AlfuCache(maxCapacity = 100) + private val fallbackLayerCache: AlfuCache[(String, String, Option[String]), Option[RemoteFallbackLayer]] = AlfuCache( + maxCapacity = 100) override def currentVersion(tracingId: String): Fox[Long] = tracingDataStore.volumes.getVersion(tracingId, mayBeEmpty = Some(true), emptyFallback = Some(0L)) @@ -113,15 +120,16 @@ class VolumeTracingService @Inject()( mappingName, editableMappingTracingId) ?~> "volumeSegmentIndex.update.failed" - def applyBucketMutatingActions(updateActions: List[BucketMutatingVolumeUpdateAction], + def applyBucketMutatingActions(annotationId: String, + updateActions: List[BucketMutatingVolumeUpdateAction], newVersion: Long, userToken: Option[String]): Fox[Unit] = for { // warning, may be called multiple times with the same version number (due to transaction management). // frontend ensures that each bucket is only updated once per transaction tracingId <- updateActions.headOption.map(_.actionTracingId).toFox - fallbackLayerOpt <- getFallbackLayer(tracingId) - tracing <- find(tracingId) ?~> "tracing.notFound" + fallbackLayerOpt <- getFallbackLayer(annotationId, tracingId, userToken) + tracing <- find(annotationId, tracingId, userToken = userToken) ?~> "tracing.notFound" segmentIndexBuffer = new VolumeSegmentIndexBuffer( tracingId, volumeSegmentIndexClient, @@ -142,7 +150,13 @@ class VolumeTracingService @Inject()( if (!tracing.getHasSegmentIndex) { Fox.failure("Cannot delete segment data for annotations without segment index.") } else - deleteSegmentData(tracingId, tracing, a, segmentIndexBuffer, newVersion, userToken) ?~> "Failed to delete segment data." + deleteSegmentData(annotationId, + tracingId, + tracing, + a, + segmentIndexBuffer, + newVersion, + userToken = userToken) ?~> "Failed to delete segment data." case _ => Fox.failure("Unknown bucket-mutating action.") } _ <- segmentIndexBuffer.flush() @@ -180,6 +194,9 @@ class VolumeTracingService @Inject()( } } yield volumeTracing + protected def takeTracing(annotation: AnnotationWithTracings, tracingId: String): Box[VolumeTracing] = + annotation.getVolume(tracingId) + override def editableMappingTracingId(tracing: VolumeTracing, tracingId: String): Option[String] = if (tracing.getHasEditableMapping) Some(tracingId) else None @@ -188,7 +205,8 @@ class VolumeTracingService @Inject()( tracing.mappingName.map(editableMappingService.getBaseMappingName).getOrElse(Fox.successful(None)) else Fox.successful(tracing.mappingName) - private def deleteSegmentData(tracingId: String, + private def deleteSegmentData(annotationId: String, + tracingId: String, volumeTracing: VolumeTracing, a: DeleteSegmentDataVolumeAction, segmentIndexBuffer: VolumeSegmentIndexBuffer, @@ -208,7 +226,7 @@ class VolumeTracingService @Inject()( Fox.serialCombined(additionalCoordinateList)(additionalCoordinates => { val mag = vec3IntFromProto(resolution) for { - fallbackLayer <- getFallbackLayer(tracingId) + fallbackLayer <- getFallbackLayer(annotationId, tracingId, userToken) bucketPositionsRaw <- volumeSegmentIndexService.getSegmentToBucketIndexWithEmptyFallbackWithoutBuffer( fallbackLayer, tracingId, @@ -258,7 +276,8 @@ class VolumeTracingService @Inject()( bool2Fox(mag.isIsotropic) } - private def revertToVolumeVersion(tracingId: String, + private def revertToVolumeVersion(annotationId: String, + tracingId: String, sourceVersion: Long, newVersion: Long, tracing: VolumeTracing, @@ -268,7 +287,7 @@ class VolumeTracingService @Inject()( val bucketStream = dataLayer.volumeBucketProvider.bucketStreamWithVersion() for { - fallbackLayer <- getFallbackLayer(tracingId) + fallbackLayer <- getFallbackLayer(annotationId, tracingId, userToken) segmentIndexBuffer = new VolumeSegmentIndexBuffer(tracingId, volumeSegmentIndexClient, newVersion, @@ -276,7 +295,7 @@ class VolumeTracingService @Inject()( fallbackLayer, dataLayer.additionalAxes, userToken) - sourceTracing <- find(tracingId, Some(sourceVersion)) + sourceTracing <- find(annotationId, tracingId, Some(sourceVersion), userToken = userToken) mappingName <- baseMappingName(sourceTracing) _ <- Fox.serialCombined(bucketStream) { case (bucketPosition, dataBeforeRevert, version) => @@ -319,7 +338,8 @@ class VolumeTracingService @Inject()( } yield sourceTracing } - def initializeWithDataMultiple(tracingId: String, + def initializeWithDataMultiple(annotationId: String, + tracingId: String, tracing: VolumeTracing, initialData: File, userToken: Option[String])(implicit mp: MessagesProvider): Fox[Set[Vec3Int]] = @@ -359,7 +379,7 @@ class VolumeTracingService @Inject()( mergedVolume.largestSegmentId.toLong, tracing.elementClass) destinationDataLayer = volumeTracingLayer(tracingId, tracing) - fallbackLayer <- getFallbackLayer(tracingId) + fallbackLayer <- getFallbackLayer(annotationId, tracingId, userToken) segmentIndexBuffer = new VolumeSegmentIndexBuffer( tracingId, volumeSegmentIndexClient, @@ -389,7 +409,8 @@ class VolumeTracingService @Inject()( } yield resolutions } - def initializeWithData(tracingId: String, + def initializeWithData(annotationId: String, + tracingId: String, tracing: VolumeTracing, initialData: File, resolutionRestrictions: ResolutionRestrictions, @@ -400,7 +421,7 @@ class VolumeTracingService @Inject()( val dataLayer = volumeTracingLayer(tracingId, tracing) val savedResolutions = new mutable.HashSet[Vec3Int]() for { - fallbackLayer <- getFallbackLayer(tracingId) + fallbackLayer <- getFallbackLayer(annotationId, tracingId, userToken) mappingName <- baseMappingName(tracing) segmentIndexBuffer = new VolumeSegmentIndexBuffer( tracingId, @@ -493,7 +514,8 @@ class VolumeTracingService @Inject()( data <- binaryDataService.handleDataRequests(requests) } yield data - def duplicate(tracingId: String, + def duplicate(annotationId: String, + tracingId: String, sourceTracing: VolumeTracing, fromTask: Boolean, datasetBoundingBox: Option[BoundingBox], @@ -506,7 +528,7 @@ class VolumeTracingService @Inject()( val tracingWithBB = addBoundingBoxFromTaskIfRequired(sourceTracing, fromTask, datasetBoundingBox) val tracingWithResolutionRestrictions = restrictMagList(tracingWithBB, resolutionRestrictions) for { - fallbackLayer <- getFallbackLayer(tracingId) + fallbackLayer <- getFallbackLayer(annotationId, tracingId, userToken) hasSegmentIndex <- VolumeSegmentIndexService.canHaveSegmentIndex(remoteDatastoreClient, fallbackLayer, userToken) newTracing = tracingWithResolutionRestrictions.copy( createdTimestamp = System.currentTimeMillis(), @@ -520,7 +542,7 @@ class VolumeTracingService @Inject()( ) _ <- bool2Fox(newTracing.resolutions.nonEmpty) ?~> "resolutionRestrictions.tooTight" newId <- save(newTracing, None, newTracing.version) - _ <- duplicateData(tracingId, sourceTracing, newId, newTracing, userToken) + _ <- duplicateData(annotationId, tracingId, sourceTracing, newId, newTracing, userToken) } yield (newId, newTracing) } @@ -540,7 +562,8 @@ class VolumeTracingService @Inject()( .withBoundingBox(datasetBoundingBox.get) } else tracing - private def duplicateData(sourceId: String, + private def duplicateData(annotationId: String, + sourceId: String, sourceTracing: VolumeTracing, destinationId: String, destinationTracing: VolumeTracing, @@ -550,7 +573,7 @@ class VolumeTracingService @Inject()( sourceDataLayer = volumeTracingLayer(sourceId, sourceTracing, isTemporaryTracing) buckets: Iterator[(BucketPosition, Array[Byte])] = sourceDataLayer.bucketProvider.bucketStream() destinationDataLayer = volumeTracingLayer(destinationId, destinationTracing) - fallbackLayer <- getFallbackLayer(sourceId) + fallbackLayer <- getFallbackLayer(annotationId, sourceId, userToken) segmentIndexBuffer = new VolumeSegmentIndexBuffer( destinationId, volumeSegmentIndexClient, @@ -645,11 +668,12 @@ class VolumeTracingService @Inject()( def volumeBucketsAreEmpty(tracingId: String): Boolean = volumeDataStore.getMultipleKeys(None, Some(tracingId), limit = Some(1))(toBox).isEmpty - def createAdHocMesh(tracingId: String, + def createAdHocMesh(annotationId: String, + tracingId: String, request: WebknossosAdHocMeshRequest, userToken: Option[String]): Fox[(Array[Float], List[Int])] = for { - tracing <- find(tracingId) ?~> "tracing.notFound" + tracing <- find(annotationId: String, tracingId, userToken = userToken) ?~> "tracing.notFound" segmentationLayer = volumeTracingLayer(tracingId, tracing, includeFallbackDataIfAvailable = true, @@ -668,9 +692,9 @@ class VolumeTracingService @Inject()( result <- adHocMeshService.requestAdHocMeshViaActor(adHocMeshRequest) } yield result - def findData(tracingId: String): Fox[Option[Vec3Int]] = + def findData(annotationId: String, tracingId: String, userToken: Option[String]): Fox[Option[Vec3Int]] = for { - tracing <- find(tracingId) ?~> "tracing.notFound" + tracing <- find(annotationId: String, tracingId, userToken = userToken) ?~> "tracing.notFound" volumeLayer = volumeTracingLayer(tracingId, tracing) bucketStream = volumeLayer.bucketProvider.bucketStream(Some(tracing.version)) bucketPosOpt = if (bucketStream.hasNext) { @@ -816,7 +840,7 @@ class VolumeTracingService @Inject()( elementClass) mergedAdditionalAxes <- Fox.box2Fox(AdditionalAxis.mergeAndAssertSameAdditionalAxes(tracings.map(t => AdditionalAxis.fromProtosAsOpt(t.additionalAxes)))) - fallbackLayer <- getFallbackLayer(tracingSelectors.head.tracingId) + fallbackLayer <- getFallbackLayer("dummyAnnotationId", tracingSelectors.head.tracingId, userToken) // TODO annotation id from selectors segmentIndexBuffer = new VolumeSegmentIndexBuffer(newId, volumeSegmentIndexClient, newVersion, @@ -842,7 +866,8 @@ class VolumeTracingService @Inject()( } } - def addSegmentIndex(tracingId: String, + def addSegmentIndex(annotationId: String, + tracingId: String, tracing: VolumeTracing, currentVersion: Long, userToken: Option[String], @@ -852,7 +877,7 @@ class VolumeTracingService @Inject()( isTemporaryTracing <- isTemporaryTracing(tracingId) sourceDataLayer = volumeTracingLayer(tracingId, tracing, isTemporaryTracing) buckets: Iterator[(BucketPosition, Array[Byte])] = sourceDataLayer.bucketProvider.bucketStream() - fallbackLayer <- getFallbackLayer(tracingId) + fallbackLayer <- getFallbackLayer(annotationId, tracingId, userToken) mappingName <- baseMappingName(tracing) segmentIndexBuffer = new VolumeSegmentIndexBuffer(tracingId, volumeSegmentIndexClient, @@ -899,7 +924,8 @@ class VolumeTracingService @Inject()( alreadyHasSegmentIndex = tracing.hasSegmentIndex.getOrElse(false) } yield canHaveSegmentIndex && !alreadyHasSegmentIndex - def importVolumeData(tracingId: String, + def importVolumeData(annotationId: String, + tracingId: String, tracing: VolumeTracing, zipFile: File, currentVersion: Int, @@ -930,7 +956,7 @@ class VolumeTracingService @Inject()( mergedVolume.largestSegmentId.toLong, tracing.elementClass) dataLayer = volumeTracingLayer(tracingId, tracing) - fallbackLayer <- getFallbackLayer(tracingId) + fallbackLayer <- getFallbackLayer(annotationId, tracingId, userToken) mappingName <- baseMappingName(tracing) segmentIndexBuffer <- Fox.successful( new VolumeSegmentIndexBuffer(tracingId, @@ -995,19 +1021,23 @@ class VolumeTracingService @Inject()( Fox.failure("Cannot merge tracings with and without editable mappings") } - def getFallbackLayer(tracingId: String): Fox[Option[RemoteFallbackLayer]] = - fallbackLayerCache.getOrLoad(tracingId, t => getFallbackLayerFromWebknossos(t)) + def getFallbackLayer(annotationId: String, + tracingId: String, + userToken: Option[String]): Fox[Option[RemoteFallbackLayer]] = + fallbackLayerCache.getOrLoad((annotationId, tracingId, userToken), + t => getFallbackLayerFromWebknossos(t._1, t._2, t._3)) - private def getFallbackLayerFromWebknossos(tracingId: String) = Fox[Option[RemoteFallbackLayer]] { - for { - tracing <- find(tracingId) - dataSource <- remoteWebknossosClient.getDataSourceForTracing(tracingId) - dataSourceId = dataSource.id - fallbackLayerName = tracing.fallbackLayer - fallbackLayer = dataSource.dataLayers - .find(_.name == fallbackLayerName.getOrElse("")) - .map(RemoteFallbackLayer.fromDataLayerAndDataSource(_, dataSourceId)) - } yield fallbackLayer - } + private def getFallbackLayerFromWebknossos(annotationId: String, tracingId: String, userToken: Option[String]) = + Fox[Option[RemoteFallbackLayer]] { + for { + tracing <- find(annotationId, tracingId, userToken = userToken) + dataSource <- remoteWebknossosClient.getDataSourceForTracing(tracingId) + dataSourceId = dataSource.id + fallbackLayerName = tracing.fallbackLayer + fallbackLayer = dataSource.dataLayers + .find(_.name == fallbackLayerName.getOrElse("")) + .map(RemoteFallbackLayer.fromDataLayerAndDataSource(_, dataSourceId)) + } yield fallbackLayer + } } diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 195cba40b8b..e033c9315af 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -5,43 +5,42 @@ # Health endpoint GET /health @com.scalableminds.webknossos.tracingstore.controllers.Application.health -POST /annotation/initialize @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.initialize(token: Option[String], annotationId: String) +POST /annotation/save @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.save(token: Option[String], annotationId: String) GET /annotation/:annotationId @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.get(token: Option[String], annotationId: String, version: Option[Long]) POST /annotation/:annotationId/update @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.update(token: Option[String], annotationId: String) POST /annotation/:annotationId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionLog(token: Option[String], annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]) GET /annotation/:annotationId/updateActionStatistics @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionStatistics(token: Option[String], annotationId: String) # Volume tracings -POST /volume/save @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.save(token: Option[String]) -POST /volume/:tracingId/initialData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.initialData(token: Option[String], tracingId: String, minResolution: Option[Int], maxResolution: Option[Int]) -POST /volume/:tracingId/initialDataMultiple @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.initialDataMultiple(token: Option[String], tracingId: String) -GET /volume/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.get(token: Option[String], tracingId: String, version: Option[Long]) -GET /volume/:tracingId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.newestVersion(token: Option[String], tracingId: String) -POST /volume/:tracingId/update @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.update(token: Option[String], tracingId: String) -GET /volume/:tracingId/allDataZip @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.allDataZip(token: Option[String], tracingId: String, volumeDataZipFormat: String, version: Option[Long], voxelSize: Option[String], voxelSizeUnit: Option[String]) -POST /volume/:tracingId/data @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.data(token: Option[String], tracingId: String) -POST /volume/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.duplicate(token: Option[String], tracingId: String, fromTask: Option[Boolean], minResolution: Option[Int], maxResolution: Option[Int], downsample: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) -POST /volume/:tracingId/adHocMesh @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.requestAdHocMesh(token: Option[String], tracingId: String) -POST /volume/:tracingId/fullMesh.stl @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.loadFullMeshStl(token: Option[String], tracingId: String) -POST /volume/:tracingId/segmentIndex/:segmentId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentIndex(token: Option[String], tracingId: String, segmentId: Long) -POST /volume/:tracingId/importVolumeData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.importVolumeData(token: Option[String], tracingId: String) -POST /volume/:tracingId/addSegmentIndex @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.addSegmentIndex(token: Option[String], tracingId: String, dryRun: Boolean) -GET /volume/:tracingId/findData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.findData(token: Option[String], tracingId: String) -GET /volume/:tracingId/agglomerateSkeleton/:agglomerateId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.agglomerateSkeleton(token: Option[String], tracingId: String, agglomerateId: Long) -POST /volume/:tracingId/makeMappingEditable @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.makeMappingEditable(token: Option[String], tracingId: String) -POST /volume/:tracingId/agglomerateGraphMinCut @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.agglomerateGraphMinCut(token: Option[String], tracingId: String) -POST /volume/:tracingId/agglomerateGraphNeighbors @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.agglomerateGraphNeighbors(token: Option[String], tracingId: String) -POST /volume/:tracingId/segmentStatistics/volume @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentVolume(token: Option[String], tracingId: String) -POST /volume/:tracingId/segmentStatistics/boundingBox @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentBoundingBox(token: Option[String], tracingId: String) -POST /volume/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getMultiple(token: Option[String]) -POST /volume/mergedFromIds @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromIds(token: Option[String], persist: Boolean) -POST /volume/mergedFromContents @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromContents(token: Option[String], persist: Boolean) +POST /volume/save @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.save(token: Option[String]) +POST /volume/:annotationId/:tracingId/initialData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.initialData(token: Option[String], annotationId: String, tracingId: String, minResolution: Option[Int], maxResolution: Option[Int]) +POST /volume/:annotationId/:tracingId/initialDataMultiple @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.initialDataMultiple(token: Option[String], annotationId: String, tracingId: String) +GET /volume/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.get(token: Option[String], annotationId: String, tracingId: String, version: Option[Long]) +GET /volume/:annotationId/:tracingId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.newestVersion(token: Option[String], annotationId: String, tracingId: String) +GET /volume/:annotationId/:tracingId/allDataZip @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.allDataZip(token: Option[String], annotationId: String, tracingId: String, volumeDataZipFormat: String, version: Option[Long], voxelSize: Option[String], voxelSizeUnit: Option[String]) +POST /volume/:annotationId/:tracingId/data @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.data(token: Option[String], annotationId: String, tracingId: String) +POST /volume/:annotationId/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.duplicate(token: Option[String], annotationId: String, tracingId: String, fromTask: Option[Boolean], minResolution: Option[Int], maxResolution: Option[Int], downsample: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) +POST /volume/:annotationId/:tracingId/adHocMesh @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.requestAdHocMesh(token: Option[String], annotationId: String, tracingId: String) +POST /volume/:annotationId/:tracingId/fullMesh.stl @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.loadFullMeshStl(token: Option[String], annotationId: String, tracingId: String) +POST /volume/:annotationId/:tracingId/segmentIndex/:segmentId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentIndex(token: Option[String], annotationId: String, tracingId: String, segmentId: Long) +POST /volume/:annotationId/:tracingId/importVolumeData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.importVolumeData(token: Option[String], annotationId: String, tracingId: String) +POST /volume/:annotationId/:tracingId/addSegmentIndex @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.addSegmentIndex(token: Option[String], annotationId: String, tracingId: String, dryRun: Boolean) +GET /volume/:annotationId/:tracingId/findData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.findData(token: Option[String], annotationId: String, tracingId: String) +GET /volume/:annotationId/:tracingId/agglomerateSkeleton/:agglomerateId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.agglomerateSkeleton(token: Option[String], annotationId: String, tracingId: String, agglomerateId: Long) +POST /volume/:annotationId/:tracingId/makeMappingEditable @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.makeMappingEditable(token: Option[String], annotationId: String, tracingId: String) +POST /volume/:annotationId/:tracingId/agglomerateGraphMinCut @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.agglomerateGraphMinCut(token: Option[String], annotationId: String, tracingId: String) +POST /volume/:annotationId/:tracingId/agglomerateGraphNeighbors @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.agglomerateGraphNeighbors(token: Option[String], annotationId: String, tracingId: String) +POST /volume/:annotationId/:tracingId/segmentStatistics/volume @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentVolume(token: Option[String], annotationId: String, tracingId: String) +POST /volume/:annotationId/:tracingId/segmentStatistics/boundingBox @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentBoundingBox(token: Option[String], annotationId: String, tracingId: String) +POST /volume/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getMultiple(token: Option[String]) +POST /volume/mergedFromIds @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromIds(token: Option[String], persist: Boolean) +POST /volume/mergedFromContents @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromContents(token: Option[String], persist: Boolean) # Editable Mappings -POST /mapping/:tracingId/update @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.updateEditableMapping(token: Option[String], tracingId: String) -GET /mapping/:tracingId/info @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingInfo(token: Option[String], tracingId: String, version: Option[Long]) -GET /mapping/:tracingId/segmentsForAgglomerate @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingSegmentIdsForAgglomerate(token: Option[String], tracingId: String, agglomerateId: Long) -POST /mapping/:tracingId/agglomeratesForSegments @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingAgglomerateIdsForSegments(token: Option[String], tracingId: String) +POST /mapping/:annotationId/:tracingId/update @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.updateEditableMapping(token: Option[String], annotationId: String, tracingId: String) +GET /mapping/:annotationId/:tracingId/info @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingInfo(token: Option[String], annotationId: String, tracingId: String, version: Option[Long]) +GET /mapping/:annotationId/:tracingId/segmentsForAgglomerate @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingSegmentIdsForAgglomerate(token: Option[String], annotationId: String, tracingId: String, agglomerateId: Long) +POST /mapping/:annotationId/:tracingId/agglomeratesForSegments @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingAgglomerateIdsForSegments(token: Option[String], annotationId: String, tracingId: String) # Zarr endpoints for volume annotations # Zarr version 2 @@ -76,9 +75,8 @@ POST /skeleton/saveMultiple @com.scalablemin POST /skeleton/mergedFromContents @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.mergedFromContents(token: Option[String], persist: Boolean) POST /skeleton/mergedFromIds @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.mergedFromIds(token: Option[String], persist: Boolean) -GET /skeleton/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.get(token: Option[String], tracingId: String, version: Option[Long]) -GET /skeleton/:tracingId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.newestVersion(token: Option[String], tracingId: String) -POST /skeleton/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.getMultiple(token: Option[String]) +GET /skeleton/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.get(token: Option[String], annotationId: String, tracingId: String, version: Option[Long]) +GET /skeleton/:annotationId/:tracingId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.newestVersion(token: Option[String], annotationId: String, tracingId: String) +POST /skeleton/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.getMultiple(token: Option[String]) -POST /skeleton/:tracingId/update @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.update(token: Option[String], tracingId: String) -POST /skeleton/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.duplicate(token: Option[String], tracingId: String, version: Option[Long], fromTask: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) +POST /skeleton/:annotationId/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.duplicate(token: Option[String], annotationId: String, tracingId: String, version: Option[Long], fromTask: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) From 0105e0e42e09bd4914bd7172039a4330d288d36e Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 27 Aug 2024 10:20:03 +0200 Subject: [PATCH 037/150] pass annotation id + token to more spots --- .../controllers/TracingController.scala | 8 +++- .../controllers/VolumeTracingController.scala | 13 +++--- .../tracings/TracingService.scala | 9 ++-- .../tracings/volume/TSFullMeshService.scala | 44 ++++++++++++------- .../VolumeSegmentStatisticsService.scala | 23 +++++----- .../volume/VolumeTracingDownsampling.scala | 7 +-- .../volume/VolumeTracingService.scala | 7 +-- 7 files changed, 69 insertions(+), 42 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala index 580593fccd8..a2db158e6a6 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala @@ -93,7 +93,9 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C log() { accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { for { - tracings <- tracingService.findMultiple(request.body, applyUpdates = true) + tracings <- tracingService.findMultiple(request.body, + applyUpdates = true, + userToken = urlOrHeaderToken(token, request)) } yield { Ok(tracings.toByteArray).as(protobufMimeType) } @@ -115,7 +117,9 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C log() { accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { for { - tracingOpts <- tracingService.findMultiple(request.body, applyUpdates = true) ?~> Messages( + tracingOpts <- tracingService.findMultiple(request.body, + applyUpdates = true, + userToken = urlOrHeaderToken(token, request)) ?~> Messages( "tracing.notFound") tracingsWithIds = tracingOpts.zip(request.body).flatMap { case (Some(tracing), Some(selector)) => Some((tracing, selector.tracingId)) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 6b5671eddaf..477f70a2c66 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -244,7 +244,8 @@ class VolumeTracingController @Inject()( newEditableMappingId, userToken ) - _ <- Fox.runIfOptionTrue(downsample)(tracingService.downsample(newId, tracingId, newTracing, userToken)) + _ <- Fox.runIfOptionTrue(downsample)( + tracingService.downsample(annotationId, newId, tracingId, newTracing, userToken)) } yield Ok(Json.toJson(newId)) } } @@ -342,11 +343,11 @@ class VolumeTracingController @Inject()( } } - def loadFullMeshStl(token: Option[String], tracingId: String): Action[FullMeshRequest] = + def loadFullMeshStl(token: Option[String], annotationId: String, tracingId: String): Action[FullMeshRequest] = Action.async(validateJson[FullMeshRequest]) { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - data: Array[Byte] <- fullMeshService.loadFor(token: Option[String], tracingId, request.body) ?~> "mesh.file.loadChunk.failed" + data: Array[Byte] <- fullMeshService.loadFor(token: Option[String], annotationId, tracingId, request.body) ?~> "mesh.file.loadChunk.failed" } yield Ok(data) } } @@ -583,7 +584,8 @@ class VolumeTracingController @Inject()( tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) mappingName <- tracingService.baseMappingName(tracing) segmentVolumes <- Fox.serialCombined(request.body.segmentIds) { segmentId => - volumeSegmentStatisticsService.getSegmentVolume(tracingId, + volumeSegmentStatisticsService.getSegmentVolume(annotationId, + tracingId, segmentId, request.body.mag, mappingName, @@ -603,7 +605,8 @@ class VolumeTracingController @Inject()( tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) mappingName <- tracingService.baseMappingName(tracing) segmentBoundingBoxes: List[BoundingBox] <- Fox.serialCombined(request.body.segmentIds) { segmentId => - volumeSegmentStatisticsService.getSegmentBoundingBox(tracingId, + volumeSegmentStatisticsService.getSegmentBoundingBox(annotationId, + tracingId, segmentId, request.body.mag, mappingName, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala index 8bba698ee1b..bf59ca54551 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala @@ -136,11 +136,14 @@ trait TracingService[T <: GeneratedMessage] def findMultiple(selectors: List[Option[TracingSelector]], useCache: Boolean = true, - applyUpdates: Boolean = false): Fox[List[Option[T]]] = + applyUpdates: Boolean = false, + userToken: Option[String]): Fox[List[Option[T]]] = Fox.combined { selectors.map { - case Some(selector) => find(selector.tracingId, selector.version, useCache, applyUpdates).map(Some(_)) - case None => Fox.successful(None) + case Some(selector) => + find("dummyAnnotationid", selector.tracingId, selector.version, useCache, applyUpdates, userToken = userToken) + .map(Some(_)) + case None => Fox.successful(None) } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala index 8ccfcfc7033..726de0f8af9 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala @@ -33,13 +33,13 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, with FullMeshHelper with LazyLogging { - def loadFor(token: Option[String], tracingId: String, fullMeshRequest: FullMeshRequest)( + def loadFor(token: Option[String], annotationId: String, tracingId: String, fullMeshRequest: FullMeshRequest)( implicit ec: ExecutionContext): Fox[Array[Byte]] = for { - tracing <- volumeTracingService.find(tracingId) ?~> "tracing.notFound" + tracing <- volumeTracingService.find(annotationId, tracingId, userToken = token) ?~> "tracing.notFound" data <- if (fullMeshRequest.meshFileName.isDefined) loadFullMeshFromMeshfile(token, tracing, tracingId, fullMeshRequest) - else loadFullMeshFromAdHoc(token, tracing, tracingId, fullMeshRequest) + else loadFullMeshFromAdHoc(token, tracing, annotationId, tracingId, fullMeshRequest) } yield data private def loadFullMeshFromMeshfile( @@ -60,6 +60,7 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, private def loadFullMeshFromAdHoc(token: Option[String], tracing: VolumeTracing, + annotationId: String, tracingId: String, fullMeshRequest: FullMeshRequest)(implicit ec: ExecutionContext): Fox[Array[Byte]] = for { @@ -68,16 +69,19 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, before = Instant.now voxelSize <- remoteDatastoreClient.voxelSizeForTracingWithCache(tracingId, token) ?~> "voxelSize.failedToFetch" verticesForChunks <- if (tracing.hasSegmentIndex.getOrElse(false)) - getAllAdHocChunksWithSegmentIndex(token, tracing, tracingId, mag, voxelSize, fullMeshRequest) + getAllAdHocChunksWithSegmentIndex(token, annotationId, tracing, tracingId, mag, voxelSize, fullMeshRequest) else - getAllAdHocChunksWithNeighborLogic(token, - tracing, - tracingId, - mag, - voxelSize, - fullMeshRequest, - fullMeshRequest.seedPosition.map(sp => VoxelPosition(sp.x, sp.y, sp.z, mag)), - adHocChunkSize) + getAllAdHocChunksWithNeighborLogic( + token, + tracing, + annotationId, + tracingId, + mag, + voxelSize, + fullMeshRequest, + fullMeshRequest.seedPosition.map(sp => VoxelPosition(sp.x, sp.y, sp.z, mag)), + adHocChunkSize + ) encoded = verticesForChunks.map(adHocMeshToStl) array = combineEncodedChunksToStl(encoded) _ = logMeshingDuration(before, "ad-hoc meshing (tracingstore)", array.length) @@ -85,13 +89,14 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, private def getAllAdHocChunksWithSegmentIndex( token: Option[String], + annotationId: String, tracing: VolumeTracing, tracingId: String, mag: Vec3Int, voxelSize: VoxelSize, fullMeshRequest: FullMeshRequest)(implicit ec: ExecutionContext): Fox[List[Array[Float]]] = for { - fallbackLayer <- volumeTracingService.getFallbackLayer(tracingId) + fallbackLayer <- volumeTracingService.getFallbackLayer(annotationId, tracingId, userToken = token) mappingName <- volumeTracingService.baseMappingName(tracing) bucketPositionsRaw: ListOfVec3IntProto <- volumeSegmentIndexService .getSegmentToBucketIndexWithEmptyFallbackWithoutBuffer( @@ -124,13 +129,14 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, fullMeshRequest.additionalCoordinates, findNeighbors = false ) - loadMeshChunkFromAdHoc(token, tracing, adHocMeshRequest, tracingId) + loadMeshChunkFromAdHoc(token, tracing, adHocMeshRequest, annotationId, tracingId) } allVertices = vertexChunksWithNeighbors.map(_._1) } yield allVertices private def getAllAdHocChunksWithNeighborLogic(token: Option[String], tracing: VolumeTracing, + annotationId: String, tracingId: String, mag: Vec3Int, voxelSize: VoxelSize, @@ -153,12 +159,17 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, fullMeshRequest.additionalCoordinates ) _ = visited += topLeft - (vertices: Array[Float], neighbors) <- loadMeshChunkFromAdHoc(token, tracing, adHocMeshRequest, tracingId) + (vertices: Array[Float], neighbors) <- loadMeshChunkFromAdHoc(token, + tracing, + adHocMeshRequest, + annotationId, + tracingId) nextPositions: List[VoxelPosition] = generateNextTopLeftsFromNeighbors(topLeft, neighbors, chunkSize, visited) _ = visited ++= nextPositions neighborVerticesNested <- Fox.serialCombined(nextPositions) { position: VoxelPosition => getAllAdHocChunksWithNeighborLogic(token, tracing, + annotationId, tracingId, mag, voxelSize, @@ -173,8 +184,9 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, private def loadMeshChunkFromAdHoc(token: Option[String], tracing: VolumeTracing, adHocMeshRequest: WebknossosAdHocMeshRequest, + annotationId: String, tracingId: String): Fox[(Array[Float], List[Int])] = if (tracing.getHasEditableMapping) editableMappingService.createAdHocMesh(tracing, tracingId, adHocMeshRequest, token) - else volumeTracingService.createAdHocMesh(tracingId, adHocMeshRequest, token) + else volumeTracingService.createAdHocMesh(annotationId, tracingId, adHocMeshRequest, token) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentStatisticsService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentStatisticsService.scala index cb12c273f53..a185e2b3735 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentStatisticsService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentStatisticsService.scala @@ -21,7 +21,8 @@ class VolumeSegmentStatisticsService @Inject()(volumeTracingService: VolumeTraci with SegmentStatistics { // Returns the segment volume (=number of voxels) in the target mag - def getSegmentVolume(tracingId: String, + def getSegmentVolume(annotationId: String, + tracingId: String, segmentId: Long, mag: Vec3Int, mappingName: Option[String], @@ -31,11 +32,12 @@ class VolumeSegmentStatisticsService @Inject()(volumeTracingService: VolumeTraci segmentId, mag, additionalCoordinates, - getBucketPositions(tracingId, mappingName, additionalCoordinates, userToken), - getTypedDataForBucketPosition(tracingId, userToken) + getBucketPositions(annotationId, tracingId, mappingName, additionalCoordinates, userToken), + getTypedDataForBucketPosition(annotationId, tracingId, userToken) ) - def getSegmentBoundingBox(tracingId: String, + def getSegmentBoundingBox(annotationId: String, + tracingId: String, segmentId: Long, mag: Vec3Int, mappingName: Option[String], @@ -45,16 +47,16 @@ class VolumeSegmentStatisticsService @Inject()(volumeTracingService: VolumeTraci segmentId, mag, additionalCoordinates, - getBucketPositions(tracingId, mappingName, additionalCoordinates, userToken), - getTypedDataForBucketPosition(tracingId, userToken) + getBucketPositions(annotationId, tracingId, mappingName, additionalCoordinates, userToken), + getTypedDataForBucketPosition(annotationId, tracingId, userToken) ) - private def getTypedDataForBucketPosition(tracingId: String, userToken: Option[String])( + private def getTypedDataForBucketPosition(annotationId: String, tracingId: String, userToken: Option[String])( bucketPosition: Vec3Int, mag: Vec3Int, additionalCoordinates: Option[Seq[AdditionalCoordinate]]) = for { - tracing <- volumeTracingService.find(tracingId) ?~> "tracing.notFound" + tracing <- volumeTracingService.find(annotationId, tracingId, userToken = userToken) ?~> "tracing.notFound" bucketData <- getVolumeDataForPositions(tracing, tracingId, mag, @@ -67,13 +69,14 @@ class VolumeSegmentStatisticsService @Inject()(volumeTracingService: VolumeTraci } yield dataTyped private def getBucketPositions( + annotationId: String, tracingId: String, mappingName: Option[String], additionalCoordinates: Option[Seq[AdditionalCoordinate]], userToken: Option[String])(segmentId: Long, mag: Vec3Int)(implicit ec: ExecutionContext) = for { - fallbackLayer <- volumeTracingService.getFallbackLayer(tracingId) - tracing <- volumeTracingService.find(tracingId) ?~> "tracing.notFound" + fallbackLayer <- volumeTracingService.getFallbackLayer(annotationId, tracingId, userToken) + tracing <- volumeTracingService.find(annotationId, tracingId, userToken = userToken) ?~> "tracing.notFound" additionalAxes = AdditionalAxis.fromProtosAsOpt(tracing.additionalAxes) allBucketPositions: ListOfVec3IntProto <- volumeSegmentIndexService .getSegmentToBucketIndexWithEmptyFallbackWithoutBuffer( diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala index bb2dd9f18a1..25f59bb4bda 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala @@ -77,7 +77,8 @@ trait VolumeTracingDownsampling protected def volumeSegmentIndexClient: FossilDBClient - protected def downsampleWithLayer(tracingId: String, + protected def downsampleWithLayer(annotationId: String, + tracingId: String, oldTracingId: String, tracing: VolumeTracing, dataLayer: VolumeTracingLayer, @@ -105,8 +106,8 @@ trait VolumeTracingDownsampling dataLayer) requiredMag } - fallbackLayer <- tracingService.getFallbackLayer(oldTracingId) // remote wk does not know the new id yet - tracing <- tracingService.find(tracingId) ?~> "tracing.notFound" + fallbackLayer <- tracingService.getFallbackLayer(annotationId, oldTracingId, userToken) // remote wk does not know the new id yet + tracing <- tracingService.find(annotationId, tracingId, userToken = userToken) ?~> "tracing.notFound" segmentIndexBuffer = new VolumeSegmentIndexBuffer(tracingId, volumeSegmentIndexClient, tracing.version, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index efeef014b49..f47bb54ba36 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -6,7 +6,6 @@ import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int} import com.scalableminds.util.io.{NamedStream, ZipIO} import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.{Fox, FoxImplicits} -import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.dataformats.wkw.WKWDataFormatHelper import com.scalableminds.webknossos.datastore.geometry.NamedBoundingBoxProto @@ -651,12 +650,14 @@ class VolumeTracingService @Inject()( toCache) } yield id - def downsample(tracingId: String, + def downsample(annotationId: String, + tracingId: String, oldTracingId: String, tracing: VolumeTracing, userToken: Option[String]): Fox[Unit] = for { - resultingResolutions <- downsampleWithLayer(tracingId, + resultingResolutions <- downsampleWithLayer(annotationId, + tracingId, oldTracingId, tracing, volumeTracingLayer(tracingId, tracing), From e42c07132acb98c7906589bb81d3d5cdb721fe4c Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 27 Aug 2024 10:24:43 +0200 Subject: [PATCH 038/150] use annotationId also in zarr streaming --- ...VolumeTracingZarrStreamingController.scala | 80 ++++++++++++------- ...alableminds.webknossos.tracingstore.routes | 42 +++++----- 2 files changed, 72 insertions(+), 50 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala index 0466da5c31c..ddc2b73a5f9 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala @@ -56,11 +56,15 @@ class VolumeTracingZarrStreamingController @Inject()( override def defaultErrorCode: Int = NOT_FOUND - def volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int): Action[AnyContent] = + def volumeTracingFolderContent(token: Option[String], + annotationId: String, + tracingId: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( + "tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) additionalFiles = if (zarrVersion == 2) List(NgffMetadata.FILENAME_DOT_ZATTRS, NgffGroupHeader.FILENAME_DOT_ZGROUP) @@ -75,11 +79,15 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def volumeTracingFolderContentJson(token: Option[String], tracingId: String, zarrVersion: Int): Action[AnyContent] = + def volumeTracingFolderContentJson(token: Option[String], + annotationId: String, + tracingId: String, + zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( + "tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto(_).toMagLiteral(allowScalar = true)) additionalFiles = if (zarrVersion == 2) List(NgffMetadata.FILENAME_DOT_ZATTRS, NgffGroupHeader.FILENAME_DOT_ZGROUP) @@ -89,13 +97,15 @@ class VolumeTracingZarrStreamingController @Inject()( } def volumeTracingMagFolderContent(token: Option[String], + annotationId: String, tracingId: String, mag: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( + "tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND @@ -112,14 +122,15 @@ class VolumeTracingZarrStreamingController @Inject()( } def volumeTracingMagFolderContentJson(token: Option[String], + annotationId: String, tracingId: String, mag: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND - + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( + "tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND @@ -128,15 +139,14 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def zArray(token: Option[String], tracingId: String, mag: String): Action[AnyContent] = Action.async { - implicit request => + def zArray(token: Option[String], annotationId: String, tracingId: String, mag: String): Action[AnyContent] = + Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND - + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( + "tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) - magParsed <- Vec3Int - .fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND + magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND cubeLength = DataLayer.bucketLength @@ -162,17 +172,17 @@ class VolumeTracingZarrStreamingController @Inject()( order = ArrayOrder.F) } yield Ok(Json.toJson(zarrHeader)) } - } + } - def zarrJsonForMag(token: Option[String], tracingId: String, mag: String): Action[AnyContent] = Action.async { - implicit request => + def zarrJsonForMag(token: Option[String], annotationId: String, tracingId: String, mag: String): Action[AnyContent] = + Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( + "tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) - magParsed <- Vec3Int - .fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND + magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND additionalAxes = AdditionalAxis.fromProtos(tracing.additionalAxes) @@ -209,12 +219,13 @@ class VolumeTracingZarrStreamingController @Inject()( ) } yield Ok(Json.toJson(zarrHeader)) } - } - - def zGroup(token: Option[String], tracingId: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { - Future(Ok(Json.toJson(NgffGroupHeader(zarr_format = 2)))) } + + def zGroup(token: Option[String], annotationId: String, tracingId: String): Action[AnyContent] = Action.async { + implicit request => + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + Future(Ok(Json.toJson(NgffGroupHeader(zarr_format = 2)))) + } } /** @@ -224,11 +235,13 @@ class VolumeTracingZarrStreamingController @Inject()( */ def zAttrs( token: Option[String], + annotationId: String, tracingId: String, ): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( + "tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) dataSource <- remoteWebknossosClient.getDataSourceForTracing(tracingId) ~> NOT_FOUND @@ -241,11 +254,13 @@ class VolumeTracingZarrStreamingController @Inject()( def zarrJson( token: Option[String], + annotationId: String, tracingId: String, ): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( + "tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) dataSource <- remoteWebknossosClient.getDataSourceForTracing(tracingId) ~> NOT_FOUND @@ -259,13 +274,15 @@ class VolumeTracingZarrStreamingController @Inject()( } def zarrSource(token: Option[String], + annotationId: String, tracingId: String, tracingName: Option[String], zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( + "tracing.notFound") ~> NOT_FOUND zarrLayer = ZarrSegmentationLayer( name = tracingName.getOrElse(tracingId), @@ -281,12 +298,17 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def rawZarrCube(token: Option[String], tracingId: String, mag: String, coordinates: String): Action[AnyContent] = + def rawZarrCube(token: Option[String], + annotationId: String, + tracingId: String, + mag: String, + coordinates: String): Action[AnyContent] = Action.async { implicit request => { accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { for { - tracing <- tracingService.find(tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( + "tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index e033c9315af..9058706cf74 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -44,29 +44,29 @@ POST /mapping/:annotationId/:tracingId/agglomeratesForSegments @c # Zarr endpoints for volume annotations # Zarr version 2 -GET /volume/zarr/json/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(token: Option[String], tracingId: String, zarrVersion: Int = 2) -GET /volume/zarr/json/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 2) -GET /volume/zarr/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int = 2) -GET /volume/zarr/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int = 2) -GET /volume/zarr/:tracingId/.zgroup @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zGroup(token: Option[String], tracingId: String) -GET /volume/zarr/:tracingId/.zattrs @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zAttrs(token: Option[String], tracingId: String) -GET /volume/zarr/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(token: Option[String], tracingId: String, tracingName: Option[String], zarrVersion: Int = 2) -GET /volume/zarr/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 2) -GET /volume/zarr/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 2) -GET /volume/zarr/:tracingId/:mag/.zarray @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zArray(token: Option[String], tracingId: String, mag: String) -GET /volume/zarr/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], tracingId: String, mag: String, coordinates: String) +GET /volume/zarr/json/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(token: Option[String], annotationId: String, tracingId: String, zarrVersion: Int = 2) +GET /volume/zarr/json/:annotationId/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(token: Option[String], annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 2) +GET /volume/zarr/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], annotationId: String, tracingId: String, zarrVersion: Int = 2) +GET /volume/zarr/:annotationId/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], annotationId: String, tracingId: String, zarrVersion: Int = 2) +GET /volume/zarr/:annotationId/:tracingId/.zgroup @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zGroup(token: Option[String], annotationId: String, tracingId: String) +GET /volume/zarr/:annotationId/:tracingId/.zattrs @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zAttrs(token: Option[String], annotationId: String, tracingId: String) +GET /volume/zarr/:annotationId/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(token: Option[String], annotationId: String, tracingId: String, tracingName: Option[String], zarrVersion: Int = 2) +GET /volume/zarr/:annotationId/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 2) +GET /volume/zarr/:annotationId/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 2) +GET /volume/zarr/:annotationId/:tracingId/:mag/.zarray @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zArray(token: Option[String], annotationId: String, tracingId: String, mag: String) +GET /volume/zarr/:annotationId/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], annotationId: String, tracingId: String, mag: String, coordinates: String) # Zarr version 3 -GET /volume/zarr3_experimental/json/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(token: Option[String], tracingId: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/json/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], tracingId: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(token: Option[String], tracingId: String, tracingName: Option[String], zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:tracingId/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJson(token: Option[String], tracingId: String) -GET /volume/zarr3_experimental/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], tracingId: String, mag: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:tracingId/:mag/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJsonForMag(token: Option[String], tracingId: String, mag: String) -GET /volume/zarr3_experimental/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], tracingId: String, mag: String, coordinates: String) +GET /volume/zarr3_experimental/json/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(token: Option[String], annotationId: String, tracingId: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/json/:annotationId/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(token: Option[String], annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], annotationId: String, tracingId: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:annotationId/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], annotationId: String, tracingId: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:annotationId/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(token: Option[String], annotationId: String, tracingId: String, tracingName: Option[String], zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:annotationId/:tracingId/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJson(token: Option[String], annotationId: String, tracingId: String) +GET /volume/zarr3_experimental/:annotationId/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:annotationId/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:annotationId/:tracingId/:mag/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJsonForMag(token: Option[String], annotationId: String, tracingId: String, mag: String) +GET /volume/zarr3_experimental/:annotationId/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], annotationId: String, tracingId: String, mag: String, coordinates: String) # Skeleton tracings POST /skeleton/save @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.save(token: Option[String]) From 48628044d455dd3992dddc41a1de6a1dc7da367e Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 27 Aug 2024 10:29:35 +0200 Subject: [PATCH 039/150] remove some unused stuff --- .../skeleton/SkeletonTracingService.scala | 28 ++----------------- .../volume/VolumeTracingService.scala | 1 + 2 files changed, 3 insertions(+), 26 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala index 15b77bf10f0..8288aaa8016 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala @@ -8,15 +8,10 @@ import com.scalableminds.webknossos.datastore.geometry.NamedBoundingBoxProto import com.scalableminds.webknossos.datastore.helpers.{ProtoGeometryImplicits, SkeletonTracingDefaults} import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis import com.scalableminds.webknossos.tracingstore.TracingStoreRedisStore -import com.scalableminds.webknossos.tracingstore.annotation.{ - AnnotationWithTracings, - LayerUpdateAction, - TSAnnotationService -} +import com.scalableminds.webknossos.tracingstore.annotation.{AnnotationWithTracings, TSAnnotationService} import com.scalableminds.webknossos.tracingstore.tracings._ -import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating._ import com.scalableminds.webknossos.tracingstore.tracings.volume.MergedVolumeStats -import net.liftweb.common.{Box, Empty, Full} +import net.liftweb.common.{Box, Full} import play.api.i18n.MessagesProvider import scala.concurrent.ExecutionContext @@ -48,25 +43,6 @@ class SkeletonTracingService @Inject()( protected def takeTracing(annotation: AnnotationWithTracings, tracingId: String): Box[SkeletonTracing] = annotation.getSkeleton(tracingId) - private def findDesiredOrNewestPossibleVersion(tracing: SkeletonTracing, - tracingId: String, - desiredVersion: Option[Long]): Fox[Long] = - /* - * Determines the newest saved version from the updates column. - * if there are no updates at all, assume tracing is brand new (possibly created from NML, - * hence the emptyFallbck tracing.version) - */ - for { - newestUpdateVersion <- tracingDataStore.skeletonUpdates.getVersion(tracingId, - mayBeEmpty = Some(true), - emptyFallback = Some(tracing.version)) - } yield { - desiredVersion match { - case None => newestUpdateVersion - case Some(desiredSome) => math.min(desiredSome, newestUpdateVersion) - } - } - def duplicate(tracing: SkeletonTracing, fromTask: Boolean, editPosition: Option[Vec3Int], diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index f47bb54ba36..2e13a9ad3a1 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -275,6 +275,7 @@ class VolumeTracingService @Inject()( bool2Fox(mag.isIsotropic) } + // TODO private def revertToVolumeVersion(annotationId: String, tracingId: String, sourceVersion: Long, From 1ce9e833dae70d9d5ab8586c7615803aacdc8e84 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 27 Aug 2024 11:38:55 +0200 Subject: [PATCH 040/150] fix cyclic injection, fix loading skeleton + volume tracing proto --- .../WKRemoteTracingStoreClient.scala | 4 +- .../AnnotationTransactionService.scala | 44 +++++++++++- .../annotation/TSAnnotationService.scala | 71 +++++++------------ .../controllers/TSAnnotationController.scala | 1 + .../tracings/TracingService.scala | 12 +--- .../skeleton/SkeletonTracingService.scala | 16 ++++- .../volume/VolumeTracingService.scala | 16 ++++- 7 files changed, 99 insertions(+), 65 deletions(-) diff --git a/app/models/annotation/WKRemoteTracingStoreClient.scala b/app/models/annotation/WKRemoteTracingStoreClient.scala index 442a0b1a2ab..d6bcf96bcdd 100644 --- a/app/models/annotation/WKRemoteTracingStoreClient.scala +++ b/app/models/annotation/WKRemoteTracingStoreClient.scala @@ -84,10 +84,10 @@ class WKRemoteTracingStoreClient( def saveAnnotationProto(annotationId: ObjectId, annotationProto: AnnotationProto): Fox[Unit] = { logger.debug("Called to save AnnotationProto." + baseInfo) - rpc(s"${tracingStore.url}/annotations/save") + rpc(s"${tracingStore.url}/tracings/annotation/save") .addQueryString("token" -> RpcTokenHolder.webknossosToken) .addQueryString("annotationId" -> annotationId.toString) - .postProto[AnnotationProto](annotationProto) + .postProto[AnnotationProto](annotationProto) // TODO why didn’t the failure bubble up? } def duplicateSkeletonTracing(skeletonTracingId: String, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala index e0b8fd651de..4301d4508d2 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala @@ -3,6 +3,12 @@ package com.scalableminds.webknossos.tracingstore.annotation import com.scalableminds.util.tools.{Fox, JsonHelper} import com.scalableminds.util.tools.Fox.bool2Fox import com.scalableminds.webknossos.tracingstore.TracingStoreRedisStore +import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} +import com.scalableminds.webknossos.tracingstore.tracings.volume.{ + BucketMutatingVolumeUpdateAction, + UpdateBucketVolumeAction, + VolumeTracingService +} import play.api.http.Status.CONFLICT import play.api.libs.json.Json @@ -13,7 +19,10 @@ import scala.concurrent.duration._ class AnnotationTransactionService @Inject()( handledGroupIdStore: TracingStoreRedisStore, // TODO: instantiate here rather than with injection, give fix namespace prefix? uncommittedUpdatesStore: TracingStoreRedisStore, - annotationService: TSAnnotationService) { + volumeTracingService: VolumeTracingService, + tracingDataStore: TracingDataStore, + annotationService: TSAnnotationService) + extends KeyValueStoreImplicits { private val transactionGroupExpiry: FiniteDuration = 24 hours private val handledGroupCacheExpiry: FiniteDuration = 24 hours @@ -156,7 +165,7 @@ class AnnotationTransactionService @Inject()( previousVersion.flatMap { prevVersion: Long => if (prevVersion + 1 == updateGroup.version) { for { - _ <- annotationService.handleUpdateGroup(annotationId, updateGroup, userToken) + _ <- handleUpdateGroup(annotationId, updateGroup, userToken) _ <- saveToHandledGroupIdStore(annotationId, updateGroup.transactionId, updateGroup.version, @@ -167,6 +176,37 @@ class AnnotationTransactionService @Inject()( } } yield newVersion + def handleUpdateGroup(annotationId: String, updateActionGroup: UpdateActionGroup, userToken: Option[String])( + implicit ec: ExecutionContext): Fox[Unit] = + for { + _ <- tracingDataStore.annotationUpdates.put(annotationId, + updateActionGroup.version, + preprocessActionsForStorage(updateActionGroup)) + bucketMutatingActions = findBucketMutatingActions(updateActionGroup) + _ <- Fox.runIf(bucketMutatingActions.nonEmpty)( + volumeTracingService + .applyBucketMutatingActions(annotationId, bucketMutatingActions, updateActionGroup.version, userToken)) + } yield () + + private def findBucketMutatingActions(updateActionGroup: UpdateActionGroup): List[BucketMutatingVolumeUpdateAction] = + updateActionGroup.actions.flatMap { + case a: BucketMutatingVolumeUpdateAction => Some(a) + case _ => None + } + + private def preprocessActionsForStorage(updateActionGroup: UpdateActionGroup): List[UpdateAction] = { + val actionsWithInfo = updateActionGroup.actions.map( + _.addTimestamp(updateActionGroup.timestamp).addAuthorId(updateActionGroup.authorId)) match { + case Nil => List[UpdateAction]() + //to the first action in the group, attach the group's info + case first :: rest => first.addInfo(updateActionGroup.info) :: rest + } + actionsWithInfo.map { + case a: UpdateBucketVolumeAction => a.transformToCompact // TODO or not? + case a => a + } + } + /* If this update group has already been “handled” (successfully saved as either committed or uncommitted), * ignore it silently. This is in case the frontend sends a retry if it believes a save to be unsuccessful * despite the backend receiving it just fine. diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index dc333ee665f..254a5e0ce40 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -19,8 +19,6 @@ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{ } import com.scalableminds.webknossos.tracingstore.tracings.volume.{ ApplyableVolumeUpdateAction, - BucketMutatingVolumeUpdateAction, - UpdateBucketVolumeAction, VolumeTracingService, VolumeUpdateAction } @@ -33,8 +31,7 @@ import javax.inject.Inject import scala.concurrent.ExecutionContext class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosClient, - tracingDataStore: TracingDataStore, - volumeTracingService: VolumeTracingService) + tracingDataStore: TracingDataStore) extends KeyValueStoreImplicits { def reportUpdates(annotationId: String, updateGroups: List[UpdateActionGroup], userToken: Option[String]): Fox[Unit] = @@ -53,37 +50,6 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl def currentMaterializableVersion(annotationId: String): Fox[Long] = tracingDataStore.annotations.getVersion(annotationId, mayBeEmpty = Some(true), emptyFallback = Some(0L)) - def handleUpdateGroup(annotationId: String, updateActionGroup: UpdateActionGroup, userToken: Option[String])( - implicit ec: ExecutionContext): Fox[Unit] = - for { - _ <- tracingDataStore.annotationUpdates.put(annotationId, - updateActionGroup.version, - preprocessActionsForStorage(updateActionGroup)) - bucketMutatingActions = findBucketMutatingActions(updateActionGroup) - _ <- Fox.runIf(bucketMutatingActions.nonEmpty)( - volumeTracingService - .applyBucketMutatingActions(annotationId, bucketMutatingActions, updateActionGroup.version, userToken)) - } yield () - - private def findBucketMutatingActions(updateActionGroup: UpdateActionGroup): List[BucketMutatingVolumeUpdateAction] = - updateActionGroup.actions.flatMap { - case a: BucketMutatingVolumeUpdateAction => Some(a) - case _ => None - } - - private def preprocessActionsForStorage(updateActionGroup: UpdateActionGroup): List[UpdateAction] = { - val actionsWithInfo = updateActionGroup.actions.map( - _.addTimestamp(updateActionGroup.timestamp).addAuthorId(updateActionGroup.authorId)) match { - case Nil => List[UpdateAction]() - //to the first action in the group, attach the group's info - case first :: rest => first.addInfo(updateActionGroup.info) :: rest - } - actionsWithInfo.map { - case a: UpdateBucketVolumeAction => a.transformToCompact // TODO or not? - case a => a - } - } - private def findPendingUpdates(annotationId: String, existingVersion: Long, desiredVersion: Long)( implicit ec: ExecutionContext): Fox[List[UpdateAction]] = if (desiredVersion == existingVersion) Fox.successful(List()) @@ -144,42 +110,55 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl def get(annotationId: String, version: Option[Long], userToken: Option[String])( implicit ec: ExecutionContext): Fox[AnnotationProto] = for { - withTracings <- getWithTracings(annotationId, version, List.empty, userToken) + withTracings <- getWithTracings(annotationId, version, List.empty, List.empty, userToken) } yield withTracings.annotation def getWithTracings(annotationId: String, version: Option[Long], - requestedTracingIds: List[String], + requestedSkeletonTracingIds: List[String], + requestedVolumeTracingIds: List[String], userToken: Option[String])(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { annotationWithVersion <- tracingDataStore.annotations.get(annotationId, version)(fromProtoBytes[AnnotationProto]) annotation = annotationWithVersion.value - updated <- applyPendingUpdates(annotation, annotationId, version, requestedTracingIds, userToken) + updated <- applyPendingUpdates(annotation, + annotationId, + version, + requestedSkeletonTracingIds, + requestedVolumeTracingIds, + userToken) } yield updated private def applyPendingUpdates( annotation: AnnotationProto, annotationId: String, targetVersionOpt: Option[Long], - requestedTracingIds: List[String], + requestedSkeletonTracingIds: List[String], + requestedVolumeTracingIds: List[String], userToken: Option[String])(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { targetVersion <- determineTargetVersion(annotation, annotationId, targetVersionOpt) updates <- findPendingUpdates(annotationId, annotation.version, targetVersion) - annotationWithTracings <- findTracingsForUpdates(annotation, updates) // TODO pass requested tracing ids + annotationWithTracings <- findTracingsForUpdates(annotation, + updates, + requestedSkeletonTracingIds, + requestedVolumeTracingIds) updated <- applyUpdates(annotationWithTracings, annotationId, updates, targetVersion, userToken) } yield updated - private def findTracingsForUpdates(annotation: AnnotationProto, updates: List[UpdateAction])( - implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = { - val skeletonTracingIds = updates.flatMap { + private def findTracingsForUpdates( + annotation: AnnotationProto, + updates: List[UpdateAction], + requestedSkeletonTracingIds: List[String], + requestedVolumeTracingIds: List[String])(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = { + val skeletonTracingIds = (updates.flatMap { case u: SkeletonUpdateAction => Some(u.actionTracingId) case _ => None - } - val volumeTracingIds = updates.flatMap { + } ++ requestedSkeletonTracingIds).distinct + val volumeTracingIds = (updates.flatMap { case u: VolumeUpdateAction => Some(u.actionTracingId) case _ => None - } + } ++ requestedVolumeTracingIds).distinct // TODO fetch editable mappings + instantiate editableMappingUpdaters/buffers if there are updates for them val editableMappingsMap: Map[String, (EditableMappingInfo, EditableMappingUpdater)] = Map.empty for { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index b6873f0fa73..2245180a963 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -32,6 +32,7 @@ class TSAnnotationController @Inject()( for { // TODO assert id does not already exist _ <- tracingDataStore.annotations.put(annotationId, 0L, request.body) + _ = logger.info(s"stored annotationProto for $annotationId") } yield Ok } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala index bf59ca54551..7250b28eb75 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala @@ -117,22 +117,12 @@ trait TracingService[T <: GeneratedMessage] def applyPendingUpdates(tracing: T, tracingId: String, targetVersion: Option[Long]): Fox[T] = Fox.successful(tracing) - protected def takeTracing(annotation: AnnotationWithTracings, tracingId: String): Box[T] - def find(annotationId: String, tracingId: String, version: Option[Long] = None, useCache: Boolean = true, applyUpdates: Boolean = false, - userToken: Option[String]): Fox[T] = - if (tracingId == TracingIds.dummyTracingId) - Fox.successful(dummyTracing) - else { - for { - annotation <- annotationService.getWithTracings(annotationId, version, List(tracingId), userToken) // TODO is applyUpdates still needed? - tracing <- takeTracing(annotation, annotationId) - } yield tracing - } + userToken: Option[String]): Fox[T] def findMultiple(selectors: List[Option[TracingSelector]], useCache: Boolean = true, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala index 8288aaa8016..e07a99558da 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala @@ -40,8 +40,20 @@ class SkeletonTracingService @Inject()( def currentVersion(tracing: SkeletonTracing): Long = tracing.version - protected def takeTracing(annotation: AnnotationWithTracings, tracingId: String): Box[SkeletonTracing] = - annotation.getSkeleton(tracingId) + def find(annotationId: String, + tracingId: String, + version: Option[Long] = None, + useCache: Boolean = true, + applyUpdates: Boolean = false, + userToken: Option[String]): Fox[SkeletonTracing] = + if (tracingId == TracingIds.dummyTracingId) + Fox.successful(dummyTracing) + else { + for { + annotation <- annotationService.getWithTracings(annotationId, version, List(tracingId), List.empty, userToken) // TODO is applyUpdates still needed? + tracing <- annotation.getSkeleton(tracingId) + } yield tracing + } def duplicate(tracing: SkeletonTracing, fromTask: Boolean, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 2e13a9ad3a1..b22c0acc604 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -193,8 +193,20 @@ class VolumeTracingService @Inject()( } } yield volumeTracing - protected def takeTracing(annotation: AnnotationWithTracings, tracingId: String): Box[VolumeTracing] = - annotation.getVolume(tracingId) + def find(annotationId: String, + tracingId: String, + version: Option[Long] = None, + useCache: Boolean = true, + applyUpdates: Boolean = false, + userToken: Option[String]): Fox[VolumeTracing] = + if (tracingId == TracingIds.dummyTracingId) + Fox.successful(dummyTracing) + else { + for { + annotation <- annotationService.getWithTracings(annotationId, version, List.empty, List(tracingId), userToken) // TODO is applyUpdates still needed? + tracing <- annotation.getVolume(tracingId) + } yield tracing + } override def editableMappingTracingId(tracing: VolumeTracing, tracingId: String): Option[String] = if (tracing.getHasEditableMapping) Some(tracingId) else None From 1124262b634f43dda7b68aef6b7d3d4ee04a1be3 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 27 Aug 2024 11:50:45 +0200 Subject: [PATCH 041/150] request tracing from new api --- frontend/javascripts/admin/admin_rest_api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts index 74159a81cb1..da21b1f6016 100644 --- a/frontend/javascripts/admin/admin_rest_api.ts +++ b/frontend/javascripts/admin/admin_rest_api.ts @@ -896,7 +896,7 @@ export async function getTracingForAnnotationType( const possibleVersionString = version != null ? `&version=${version}` : ""; const tracingArrayBuffer = await doWithToken((token) => Request.receiveArraybuffer( - `${annotation.tracingStore.url}/tracings/${tracingType}/${tracingId}?token=${token}${possibleVersionString}`, + `${annotation.tracingStore.url}/tracings/${tracingType}/${annotation.id}/${tracingId}?token=${token}${possibleVersionString}`, { headers: { Accept: "application/x-protobuf", From 9975bf2575eca8b223b25d715d50018c4ec090dc Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 28 Aug 2024 11:48:35 +0200 Subject: [PATCH 042/150] fix report tracing updates --- app/controllers/AnnotationController.scala | 16 ++++++++++------ app/controllers/LegacyApiController.scala | 2 +- .../WKRemoteTracingStoreController.scala | 11 ++++++----- conf/webknossos.latest.routes | 4 ++-- conf/webknossos.versioned.routes | 4 ++-- .../tracingstore/TSRemoteWebknossosClient.scala | 3 ++- .../annotation/TSAnnotationService.scala | 2 +- .../controllers/VolumeTracingController.scala | 2 +- 8 files changed, 25 insertions(+), 19 deletions(-) diff --git a/app/controllers/AnnotationController.scala b/app/controllers/AnnotationController.scala index 222a3d3a21b..683022523a1 100755 --- a/app/controllers/AnnotationController.scala +++ b/app/controllers/AnnotationController.scala @@ -76,7 +76,7 @@ class AnnotationController @Inject()( // For Task and Explorational annotations, id is an annotation id. For CompoundTask, id is a task id. For CompoundProject, id is a project id. For CompoundTaskType, id is a task type id id: String, // Timestamp in milliseconds (time at which the request is sent) - timestamp: Long): Action[AnyContent] = sil.UserAwareAction.async { implicit request => + timestamp: Option[Long]): Action[AnyContent] = sil.UserAwareAction.async { implicit request => log() { val notFoundMessage = if (request.identity.isEmpty) "annotation.notFound.considerLoggingIn" else "annotation.notFound" @@ -89,10 +89,14 @@ class AnnotationController @Inject()( js <- annotationService .publicWrites(annotation, request.identity, Some(restrictions)) ?~> "annotation.write.failed" _ <- Fox.runOptional(request.identity) { user => - if (typedTyp == AnnotationType.Task || typedTyp == AnnotationType.Explorational) { - timeSpanService - .logUserInteractionIfTheyArePotentialContributor(Instant(timestamp), user, annotation) // log time when a user starts working - } else Fox.successful(()) + Fox.runOptional(timestamp) { timestampDefined => + if (typedTyp == AnnotationType.Task || typedTyp == AnnotationType.Explorational) { + timeSpanService.logUserInteractionIfTheyArePotentialContributor( + Instant(timestampDefined), + user, + annotation) // log time when a user starts working + } else Fox.successful(()) + } } _ = Fox.runOptional(request.identity)(user => userDAO.updateLastActivity(user._id)) _ = request.identity.foreach { user => @@ -104,7 +108,7 @@ class AnnotationController @Inject()( def infoWithoutType(id: String, // Timestamp in milliseconds (time at which the request is sent - timestamp: Long): Action[AnyContent] = sil.UserAwareAction.async { implicit request => + timestamp: Option[Long]): Action[AnyContent] = sil.UserAwareAction.async { implicit request => log() { for { annotation <- provider.provideAnnotation(id, request.identity) ~> NOT_FOUND diff --git a/app/controllers/LegacyApiController.scala b/app/controllers/LegacyApiController.scala index 2946ec1ddfa..725a0362f45 100644 --- a/app/controllers/LegacyApiController.scala +++ b/app/controllers/LegacyApiController.scala @@ -107,7 +107,7 @@ class LegacyApiController @Inject()(annotationController: AnnotationController, } yield adaptedResult } - def annotationInfoV4(typ: String, id: String, timestamp: Long): Action[AnyContent] = sil.SecuredAction.async { + def annotationInfoV4(typ: String, id: String, timestamp: Option[Long]): Action[AnyContent] = sil.SecuredAction.async { implicit request => for { _ <- Fox.successful(logVersioned(request)) diff --git a/app/controllers/WKRemoteTracingStoreController.scala b/app/controllers/WKRemoteTracingStoreController.scala index 6d4ac54be1a..14b5df23000 100644 --- a/app/controllers/WKRemoteTracingStoreController.scala +++ b/app/controllers/WKRemoteTracingStoreController.scala @@ -25,7 +25,7 @@ import play.api.i18n.Messages import play.api.libs.json.Json import play.api.mvc.{Action, AnyContent, PlayBodyParsers} import security.{WebknossosBearerTokenAuthenticatorService, WkSilhouetteEnvironment} -import utils.WkConf +import utils.{ObjectId, WkConf} import scala.concurrent.ExecutionContext @@ -56,12 +56,13 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore tracingStoreService.validateAccess(name, key) { _ => val report = request.body for { - annotation <- annotationDAO.findOneByTracingId(report.tracingId) + annotationId <- ObjectId.fromString(report.annotationId) + annotation <- annotationDAO.findOne(annotationId) _ <- ensureAnnotationNotFinished(annotation) _ <- annotationDAO.updateModified(annotation._id, Instant.now) - _ <- Fox.runOptional(report.statistics) { statistics => - annotationLayerDAO.updateStatistics(annotation._id, report.tracingId, statistics) - } + /*_ <- Fox.runOptional(report.statistics) { statistics => + annotationLayerDAO.updateStatistics(annotation._id, annotationId, statistics) + }*/ // TODO stats per tracing id userBox <- bearerTokenService.userForTokenOpt(report.userToken).futureBox trackTime = (report.significantChangesCount > 0 || !wkConf.WebKnossos.User.timeTrackingOnlyWithSignificantChanges) _ <- Fox.runOptional(userBox)(user => diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes index 0786fb5ac2c..c7a1babaefc 100644 --- a/conf/webknossos.latest.routes +++ b/conf/webknossos.latest.routes @@ -142,7 +142,7 @@ PUT /annotations/:typ/:id/reset PATCH /annotations/:typ/:id/transfer controllers.AnnotationController.transfer(typ: String, id: String) PATCH /annotations/:typ/:id/editLockedState controllers.AnnotationController.editLockedState(typ: String, id: String, isLockedByOwner: Boolean) -GET /annotations/:id/info controllers.AnnotationController.infoWithoutType(id: String, timestamp: Long) +GET /annotations/:id/info controllers.AnnotationController.infoWithoutType(id: String, timestamp: Option[Long]) PATCH /annotations/:id/makeHybrid controllers.AnnotationController.makeHybridWithoutType(id: String, fallbackLayerName: Option[String]) PATCH /annotations/:id/downsample controllers.AnnotationController.downsampleWithoutType(id: String, tracingId: String) PATCH /annotations/:id/addAnnotationLayer controllers.AnnotationController.addAnnotationLayerWithoutType(id: String) @@ -153,7 +153,7 @@ GET /annotations/:id/download POST /annotations/:id/acquireMutex controllers.AnnotationController.tryAcquiringAnnotationMutex(id: String) PATCH /annotations/addSegmentIndicesToAll controllers.AnnotationController.addSegmentIndicesToAll(parallelBatchCount: Int, dryRun: Boolean, skipTracings: Option[String]) -GET /annotations/:typ/:id/info controllers.AnnotationController.info(typ: String, id: String, timestamp: Long) +GET /annotations/:typ/:id/info controllers.AnnotationController.info(typ: String, id: String, timestamp: Option[Long]) PATCH /annotations/:typ/:id/makeHybrid controllers.AnnotationController.makeHybrid(typ: String, id: String, fallbackLayerName: Option[String]) PATCH /annotations/:typ/:id/downsample controllers.AnnotationController.downsample(typ: String, id: String, tracingId: String) PATCH /annotations/:typ/:id/addAnnotationLayer controllers.AnnotationController.addAnnotationLayer(typ: String, id: String) diff --git a/conf/webknossos.versioned.routes b/conf/webknossos.versioned.routes index 8447aa2a9e6..28c9ca77314 100644 --- a/conf/webknossos.versioned.routes +++ b/conf/webknossos.versioned.routes @@ -38,7 +38,7 @@ GET /v4/datasets/:organizationName/:datasetName co GET /v4/datasets/:organizationName/:datasetName/isValidNewName controllers.LegacyApiController.assertValidNewNameV5(organizationName: String, datasetName: String) # v4: support changes to v5 -GET /v4/annotations/:typ/:id/info controllers.LegacyApiController.annotationInfoV4(typ: String, id: String, timestamp: Long) +GET /v4/annotations/:typ/:id/info controllers.LegacyApiController.annotationInfoV4(typ: String, id: String, timestamp: Option[Long]) PATCH /v4/annotations/:typ/:id/finish controllers.LegacyApiController.annotationFinishV4(typ: String, id: String, timestamp: Long) POST /v4/annotations/:typ/:id/merge/:mergedTyp/:mergedId controllers.LegacyApiController.annotationMergeV4(typ: String, id: String, mergedTyp: String, mergedId: String) PATCH /v4/annotations/:typ/:id/edit controllers.LegacyApiController.annotationEditV4(typ: String, id: String) @@ -64,7 +64,7 @@ GET /v3/datasets/:organizationName/:datasetName/isValidNewName co # v3: support changes to v5 PATCH /v3/annotations/:typ/:id/finish controllers.LegacyApiController.annotationFinishV4(typ: String, id: String, timestamp: Long) -GET /v3/annotations/:typ/:id/info controllers.LegacyApiController.annotationInfoV4(typ: String, id: String, timestamp: Long) +GET /v3/annotations/:typ/:id/info controllers.LegacyApiController.annotationInfoV4(typ: String, id: String, timestamp: Option[Long]) POST /v3/annotations/:typ/:id/merge/:mergedTyp/:mergedId controllers.LegacyApiController.annotationMergeV4(typ: String, id: String, mergedTyp: String, mergedId: String) PATCH /v3/annotations/:typ/:id/edit controllers.LegacyApiController.annotationEditV4(typ: String, id: String) POST /v3/annotations/:typ/:id/duplicate controllers.LegacyApiController.annotationDuplicateV4(typ: String, id: String) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala index 6174fb4be91..b229f2a6cbd 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala @@ -19,7 +19,8 @@ import play.api.libs.ws.WSResponse import scala.concurrent.ExecutionContext -case class TracingUpdatesReport(tracingId: String, +case class TracingUpdatesReport(annotationId: String, + // TODO stats per tracing id? timestamps: List[Instant], statistics: Option[JsObject], significantChangesCount: Int, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 254a5e0ce40..e51feb2b819 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -40,7 +40,7 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl TracingUpdatesReport( annotationId, timestamps = updateGroups.map(g => Instant(g.timestamp)), - statistics = updateGroups.flatMap(_.stats).lastOption, + statistics = updateGroups.flatMap(_.stats).lastOption, // TODO statistics per tracing/layer significantChangesCount = updateGroups.map(_.significantChangesCount).sum, viewChangesCount = updateGroups.map(_.viewChangesCount).sum, userToken diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 477f70a2c66..8621dbada90 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -477,7 +477,7 @@ class VolumeTracingController @Inject()( updateGroup <- request.body.headOption.toFox _ <- bool2Fox(updateGroup.version == currentVersion + 1) ?~> "version mismatch" report = TracingUpdatesReport( - tracingId, + annotationId, // TODO integrate all of this into annotation update timestamps = List(Instant(updateGroup.timestamp)), statistics = None, significantChangesCount = updateGroup.actions.length, From 29a2f47c45511589af431369f876404da5adbdfc Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 28 Aug 2024 13:59:53 +0200 Subject: [PATCH 043/150] fix ambiguous update action keys --- .../annotation/TSAnnotationService.scala | 20 +++++++++++-------- .../annotation/UpdateActions.scala | 11 +++++----- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index e51feb2b819..b5102985d98 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -24,6 +24,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ } import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingUpdatesReport} +import com.typesafe.scalalogging.LazyLogging import net.liftweb.common.{Empty, Full} import play.api.libs.json.{JsObject, JsValue, Json} @@ -32,7 +33,8 @@ import scala.concurrent.ExecutionContext class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosClient, tracingDataStore: TracingDataStore) - extends KeyValueStoreImplicits { + extends KeyValueStoreImplicits + with LazyLogging { def reportUpdates(annotationId: String, updateGroups: List[UpdateActionGroup], userToken: Option[String]): Fox[Unit] = for { @@ -75,7 +77,7 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl case a: UpdateMetadataAnnotationUpdateAction => Fox.successful(annotationWithTracings.updateMetadata(a)) case a: SkeletonUpdateAction => - annotationWithTracings.applySkeletonAction(a) + annotationWithTracings.applySkeletonAction(a) ?~> "applySkeletonAction.failed" case a: ApplyableVolumeUpdateAction => annotationWithTracings.applyVolumeAction(a) case a: EditableMappingUpdateAction => @@ -119,14 +121,14 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl requestedVolumeTracingIds: List[String], userToken: Option[String])(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { - annotationWithVersion <- tracingDataStore.annotations.get(annotationId, version)(fromProtoBytes[AnnotationProto]) + annotationWithVersion <- tracingDataStore.annotations.get(annotationId, version)(fromProtoBytes[AnnotationProto]) ?~> "getAnnotation.failed" annotation = annotationWithVersion.value updated <- applyPendingUpdates(annotation, annotationId, version, requestedSkeletonTracingIds, requestedVolumeTracingIds, - userToken) + userToken) ?~> "applyUpdates.failed" } yield updated private def applyPendingUpdates( @@ -137,13 +139,13 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl requestedVolumeTracingIds: List[String], userToken: Option[String])(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { - targetVersion <- determineTargetVersion(annotation, annotationId, targetVersionOpt) - updates <- findPendingUpdates(annotationId, annotation.version, targetVersion) + targetVersion <- determineTargetVersion(annotation, annotationId, targetVersionOpt) ?~> "determineTargetVersion.failed" + updates <- findPendingUpdates(annotationId, annotation.version, targetVersion) ?~> "findPendingUpdates.failed" annotationWithTracings <- findTracingsForUpdates(annotation, updates, requestedSkeletonTracingIds, - requestedVolumeTracingIds) - updated <- applyUpdates(annotationWithTracings, annotationId, updates, targetVersion, userToken) + requestedVolumeTracingIds) ?~> "findTracingsForUpdates.failed" + updated <- applyUpdates(annotationWithTracings, annotationId, updates, targetVersion, userToken) ?~> "applyUpdates.inner.failed" } yield updated private def findTracingsForUpdates( @@ -161,6 +163,7 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl } ++ requestedVolumeTracingIds).distinct // TODO fetch editable mappings + instantiate editableMappingUpdaters/buffers if there are updates for them val editableMappingsMap: Map[String, (EditableMappingInfo, EditableMappingUpdater)] = Map.empty + logger.info(s"fetching volumes ${volumeTracingIds} and skeletons $skeletonTracingIds") for { skeletonTracings <- Fox.serialCombined(skeletonTracingIds)( id => @@ -170,6 +173,7 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl id => tracingDataStore.volumes .get[VolumeTracing](id, Some(annotation.version), mayBeEmpty = Some(true))(fromProtoBytes[VolumeTracing])) + _ = logger.info(s"fetched ${skeletonTracings.length} skeletons and ${volumeTracings.length} volumes") skeletonTracingsMap: Map[String, Either[SkeletonTracing, VolumeTracing]] = skeletonTracingIds .zip(skeletonTracings.map(versioned => Left[SkeletonTracing, VolumeTracing](versioned.value))) .toMap diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala index ee0cfaf4820..d0e00c49819 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala @@ -93,7 +93,8 @@ object UpdateAction { case "updateUserBoundingBoxVisibility" => deserialize[UpdateUserBoundingBoxVisibilityVolumeAction](jsonValue) case "removeFallbackLayer" => deserialize[RemoveFallbackLayerVolumeAction](jsonValue) case "importVolumeTracing" => deserialize[ImportVolumeDataVolumeAction](jsonValue) - case "updateTdCamera" => deserialize[UpdateTdCameraVolumeAction](jsonValue) + case "updateTdCameraSkeleton" => deserialize[UpdateTdCameraSkeletonAction](jsonValue) // TODO deduplicate? + case "updateTdCameraVolume" => deserialize[UpdateTdCameraVolumeAction](jsonValue) case "createSegment" => deserialize[CreateSegmentVolumeAction](jsonValue) case "updateSegment" => deserialize[UpdateSegmentVolumeAction](jsonValue) case "updateSegmentGroups" => deserialize[UpdateSegmentGroupsVolumeAction](jsonValue) @@ -150,7 +151,7 @@ object UpdateAction { case s: UpdateTreeGroupsSkeletonAction => Json.obj("name" -> "updateTreeGroups", "value" -> Json.toJson(s)(UpdateTreeGroupsSkeletonAction.jsonFormat)) case s: UpdateTracingSkeletonAction => - Json.obj("name" -> "updateTracing", "value" -> Json.toJson(s)(UpdateTracingSkeletonAction.jsonFormat)) + Json.obj("name" -> "updateSkeletonTracing", "value" -> Json.toJson(s)(UpdateTracingSkeletonAction.jsonFormat)) case s: RevertToVersionSkeletonAction => Json.obj("name" -> "revertToVersion", "value" -> Json.toJson(s)(RevertToVersionSkeletonAction.jsonFormat)) case s: UpdateTreeVisibilitySkeletonAction => @@ -169,13 +170,13 @@ object UpdateAction { Json.obj("name" -> "updateUserBoundingBoxVisibility", "value" -> Json.toJson(s)(UpdateUserBoundingBoxVisibilitySkeletonAction.jsonFormat)) case s: UpdateTdCameraSkeletonAction => - Json.obj("name" -> "updateTdCamera", "value" -> Json.toJson(s)(UpdateTdCameraSkeletonAction.jsonFormat)) + Json.obj("name" -> "updateTdCameraSkeleton", "value" -> Json.toJson(s)(UpdateTdCameraSkeletonAction.jsonFormat)) // Volume case s: UpdateBucketVolumeAction => Json.obj("name" -> "updateBucket", "value" -> Json.toJson(s)(UpdateBucketVolumeAction.jsonFormat)) case s: UpdateTracingVolumeAction => - Json.obj("name" -> "updateTracing", "value" -> Json.toJson(s)(UpdateTracingVolumeAction.jsonFormat)) + Json.obj("name" -> "updateVolumeTracing", "value" -> Json.toJson(s)(UpdateTracingVolumeAction.jsonFormat)) case s: UpdateUserBoundingBoxesVolumeAction => Json.obj("name" -> "updateUserBoundingBoxes", "value" -> Json.toJson(s)(UpdateUserBoundingBoxesVolumeAction.jsonFormat)) @@ -187,7 +188,7 @@ object UpdateAction { case s: ImportVolumeDataVolumeAction => Json.obj("name" -> "importVolumeTracing", "value" -> Json.toJson(s)(ImportVolumeDataVolumeAction.jsonFormat)) case s: UpdateTdCameraVolumeAction => - Json.obj("name" -> "updateTdCamera", "value" -> Json.toJson(s)(UpdateTdCameraVolumeAction.jsonFormat)) + Json.obj("name" -> "updateTdCameraVolume", "value" -> Json.toJson(s)(UpdateTdCameraVolumeAction.jsonFormat)) case s: CreateSegmentVolumeAction => Json.obj("name" -> "createSegment", "value" -> Json.toJson(s)(CreateSegmentVolumeAction.jsonFormat)) case s: UpdateSegmentVolumeAction => From b26e3b29d52dd9802bd84d2509936e53f5d7afd1 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Thu, 29 Aug 2024 16:49:05 +0200 Subject: [PATCH 044/150] adapt frontend to changed save route (only for skeleton; still buggy) --- .../oxalis/model/reducers/save_reducer.ts | 12 +++++++++++- .../javascripts/oxalis/model/sagas/save_saga.ts | 4 +++- .../oxalis/model/sagas/update_actions.ts | 13 +++++++++++-- frontend/javascripts/oxalis/store.ts | 4 ++-- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts index 7460f0e8de9..7239f5e3426 100644 --- a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts +++ b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts @@ -18,6 +18,7 @@ import { } from "oxalis/model/reducers/volumetracing_reducer_helpers"; import Date from "libs/date"; import * as Utils from "libs/utils"; +import { UpdateActionWithTracingId } from "../sagas/update_actions"; // These update actions are not idempotent. Having them // twice in the save queue causes a corruption of the current annotation. @@ -164,7 +165,16 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState { transactionGroupIndex, timestamp: Date.now(), authorId: activeUser.id, - actions, + actions: actions.map( + (innerAction) => + ({ + ...innerAction, + value: { + ...innerAction.value, + actionTracingId: action.tracingId, + }, + }) as UpdateActionWithTracingId, + ), stats, // Redux Action Log context for debugging purposes. info: actionLogInfo, diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts index 4a85d54f119..79b60aedc6f 100644 --- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts @@ -178,6 +178,7 @@ export function* sendRequestToServer( const { version, type } = yield* select((state) => selectTracing(state, saveQueueType, tracingId), ); + const annotationId = yield* select((state) => state.tracing.annotationId); const tracingStoreUrl = yield* select((state) => state.tracing.tracingStore.url); let versionIncrement; [compactedSaveQueue, versionIncrement] = addVersionNumbers(compactedSaveQueue, version); @@ -191,7 +192,8 @@ export function* sendRequestToServer( const startTime = Date.now(); yield* call( sendRequestWithToken, - `${tracingStoreUrl}/tracings/${type}/${tracingId}/update?token=`, + + `${tracingStoreUrl}/tracings/annotation/${annotationId}/update?token=`, { method: "POST", data: compactedSaveQueue, diff --git a/frontend/javascripts/oxalis/model/sagas/update_actions.ts b/frontend/javascripts/oxalis/model/sagas/update_actions.ts index 7df4a0179e2..5de4a15e402 100644 --- a/frontend/javascripts/oxalis/model/sagas/update_actions.ts +++ b/frontend/javascripts/oxalis/model/sagas/update_actions.ts @@ -78,6 +78,13 @@ export type UpdateAction = | UpdateMappingNameUpdateAction | SplitAgglomerateUpdateAction | MergeAgglomerateUpdateAction; + +export type UpdateActionWithTracingId = UpdateAction & { + value: UpdateAction["value"] & { + actionTracingId: string; + }; +}; + // This update action is only created in the frontend for display purposes type CreateTracingUpdateAction = { name: "createTracing"; @@ -107,6 +114,8 @@ type AddServerValuesFn = (arg0: T) => T & { type AsServerAction = ReturnType>; +// When the server delivers update actions (e.g., when requesting the version history +// of an annotation), ServerUpdateActions are sent which include some additional information. export type ServerUpdateAction = AsServerAction< | UpdateAction // These two actions are never sent by the frontend and, therefore, don't exist in the UpdateAction type @@ -260,7 +269,7 @@ export function updateSkeletonTracing( zoomLevel: number, ) { return { - name: "updateTracing", + name: "updateSkeletonTracing", value: { activeNode: tracing.activeNodeId, editPosition, @@ -411,7 +420,7 @@ export function removeFallbackLayer() { } export function updateTdCamera() { return { - name: "updateTdCamera", + name: "updateTdCameraSkeleton", value: {}, } as const; } diff --git a/frontend/javascripts/oxalis/store.ts b/frontend/javascripts/oxalis/store.ts index f449ef51634..56489f14126 100644 --- a/frontend/javascripts/oxalis/store.ts +++ b/frontend/javascripts/oxalis/store.ts @@ -49,7 +49,7 @@ import type { } from "oxalis/constants"; import type { BLEND_MODES, ControlModeEnum } from "oxalis/constants"; import type { Matrix4x4 } from "libs/mjs"; -import type { UpdateAction } from "oxalis/model/sagas/update_actions"; +import type { UpdateAction, UpdateActionWithTracingId } from "oxalis/model/sagas/update_actions"; import AnnotationReducer from "oxalis/model/reducers/annotation_reducer"; import DatasetReducer from "oxalis/model/reducers/dataset_reducer"; import type DiffableMap from "libs/diffable_map"; @@ -439,7 +439,7 @@ export type SaveQueueEntry = { version: number; timestamp: number; authorId: string; - actions: Array; + actions: Array; transactionId: string; transactionGroupCount: number; transactionGroupIndex: number; From 44ca6129090f507f22e3c71b60a0376f415a7703 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 12 Sep 2024 11:38:19 +0200 Subject: [PATCH 045/150] use annotationUpdates to get materializableVersion --- frontend/javascripts/oxalis/model/sagas/update_actions.ts | 2 +- .../tracingstore/annotation/TSAnnotationService.scala | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/javascripts/oxalis/model/sagas/update_actions.ts b/frontend/javascripts/oxalis/model/sagas/update_actions.ts index 209f399fc15..702a918260a 100644 --- a/frontend/javascripts/oxalis/model/sagas/update_actions.ts +++ b/frontend/javascripts/oxalis/model/sagas/update_actions.ts @@ -301,7 +301,7 @@ export function updateVolumeTracing( zoomLevel: number, ) { return { - name: "updateTracing", + name: "updateVolumeTracing", value: { activeSegmentId: tracing.activeCellId, editPosition: position, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index b5102985d98..f41824d1dfa 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -50,6 +50,9 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl } yield () def currentMaterializableVersion(annotationId: String): Fox[Long] = + tracingDataStore.annotationUpdates.getVersion(annotationId, mayBeEmpty = Some(true), emptyFallback = Some(0L)) + + def currentMaterializedVersion(annotationId: String): Fox[Long] = tracingDataStore.annotations.getVersion(annotationId, mayBeEmpty = Some(true), emptyFallback = Some(0L)) private def findPendingUpdates(annotationId: String, existingVersion: Long, desiredVersion: Long)( From 821377a624bf2a9d73737e70ef9f87ce70a8969e Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 12 Sep 2024 13:25:24 +0200 Subject: [PATCH 046/150] update tracing proto version --- .../annotation/AnnotationTransactionService.scala | 7 ++++++- .../tracingstore/annotation/AnnotationWithTracings.scala | 9 +++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala index 4301d4508d2..3e0efeac89a 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala @@ -9,6 +9,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ UpdateBucketVolumeAction, VolumeTracingService } +import com.typesafe.scalalogging.LazyLogging import play.api.http.Status.CONFLICT import play.api.libs.json.Json @@ -22,7 +23,8 @@ class AnnotationTransactionService @Inject()( volumeTracingService: VolumeTracingService, tracingDataStore: TracingDataStore, annotationService: TSAnnotationService) - extends KeyValueStoreImplicits { + extends KeyValueStoreImplicits + with LazyLogging { private val transactionGroupExpiry: FiniteDuration = 24 hours private val handledGroupCacheExpiry: FiniteDuration = 24 hours @@ -161,6 +163,9 @@ class AnnotationTransactionService @Inject()( for { _ <- annotationService.reportUpdates(annotationId, updateGroups, userToken) currentCommittedVersion: Fox[Long] = annotationService.currentMaterializableVersion(annotationId) + _ = logger.info(s"trying to commit ${updateGroups + .map(_.actions.length) + .sum} actions in ${updateGroups.length} groups (versions ${updateGroups.map(_.version).mkString(",")}") newVersion <- updateGroups.foldLeft(currentCommittedVersion) { (previousVersion, updateGroup) => previousVersion.flatMap { prevVersion: Long => if (prevVersion + 1 == updateGroup.version) { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index 83ca1ac6ece..d1dd18cf044 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -74,8 +74,13 @@ case class AnnotationWithTracings( tracingsById, editableMappingsByTracingId) - def withVersion(newVersion: Long): AnnotationWithTracings = - AnnotationWithTracings(annotation.copy(version = newVersion), tracingsById, editableMappingsByTracingId) // TODO also update version in tracings? + def withVersion(newVersion: Long): AnnotationWithTracings = { + val tracingsUpdated = tracingsById.view.mapValues { + case Left(t: SkeletonTracing) => Left(t.withVersion(newVersion)) + case Right(t: VolumeTracing) => Right(t.withVersion(newVersion)) + } + AnnotationWithTracings(annotation.copy(version = newVersion), tracingsUpdated.toMap, editableMappingsByTracingId) + } def applySkeletonAction(a: SkeletonUpdateAction)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { From 6ea6d654337e777411ea39b11ba99da731f9dfc6 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 16 Sep 2024 09:22:11 +0200 Subject: [PATCH 047/150] fix volume saving (single, layer, brush+move only) --- .../oxalis/model/bucket_data_handling/wkstore_adapter.ts | 3 ++- .../model/helpers/compaction/compact_save_queue.ts | 5 ++++- .../javascripts/oxalis/model/reducers/save_reducer.ts | 7 +++++-- frontend/javascripts/oxalis/model/sagas/save_saga.ts | 3 +++ .../javascripts/oxalis/model/sagas/update_actions.ts | 2 +- frontend/javascripts/test/helpers/saveHelpers.ts | 4 +++- .../annotation/AnnotationTransactionService.scala | 9 ++++----- .../tracingstore/annotation/TSAnnotationService.scala | 5 ++++- .../tracingstore/annotation/UpdateActions.scala | 2 ++ .../tracings/volume/VolumeTracingService.scala | 6 ++++-- .../tracings/volume/VolumeUpdateActions.scala | 7 +++---- 11 files changed, 35 insertions(+), 18 deletions(-) diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.ts b/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.ts index 4db8385e75b..963ca797e1c 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.ts +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.ts @@ -110,7 +110,8 @@ export async function requestWithFallback( optLayerName || layerInfo.name }`; - const getTracingStoreUrl = () => `${tracingStoreHost}/tracings/volume/${layerInfo.name}`; + const getTracingStoreUrl = () => + `${tracingStoreHost}/tracings/volume/${state.tracing.annotationId}/${layerInfo.name}`; const maybeVolumeTracing = "tracingId" in layerInfo && layerInfo.tracingId != null diff --git a/frontend/javascripts/oxalis/model/helpers/compaction/compact_save_queue.ts b/frontend/javascripts/oxalis/model/helpers/compaction/compact_save_queue.ts index 6ab8bab4525..710ce9f273a 100644 --- a/frontend/javascripts/oxalis/model/helpers/compaction/compact_save_queue.ts +++ b/frontend/javascripts/oxalis/model/helpers/compaction/compact_save_queue.ts @@ -5,7 +5,10 @@ function removeAllButLastUpdateTracingAction(updateActionsBatches: Array batch.actions.length === 1 && batch.actions[0].name === "updateTracing", + (batch) => + batch.actions.length === 1 && + (batch.actions[0].name === "updateSkeletonTracing" || + batch.actions[0].name === "updateVolumeTracing"), ); return _.without(updateActionsBatches, ...updateTracingOnlyBatches.slice(0, -1)); } diff --git a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts index 7239f5e3426..a1d1ccf668e 100644 --- a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts +++ b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts @@ -18,7 +18,7 @@ import { } from "oxalis/model/reducers/volumetracing_reducer_helpers"; import Date from "libs/date"; import * as Utils from "libs/utils"; -import { UpdateActionWithTracingId } from "../sagas/update_actions"; +import type { UpdateActionWithTracingId } from "../sagas/update_actions"; // These update actions are not idempotent. Having them // twice in the save queue causes a corruption of the current annotation. @@ -140,7 +140,10 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState { return state; } // Only report tracing statistics, if a "real" update to the tracing happened - const stats = _.some(action.items, (ua) => ua.name !== "updateTracing") + const stats = _.some( + action.items, + (ua) => ua.name !== "updateSkeletonTracing" && ua.name !== "updateVolumeTracing", + ) ? getStats(state.tracing, action.saveQueueType, action.tracingId) : null; const { activeUser } = state; diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts index 3382afb0b45..b51dd0152dc 100644 --- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts @@ -371,9 +371,12 @@ export function performDiffTracing( ); } + /* + TODO: restore this update action (decide how to handle it, does it belong to skeleton or volume or something else?) if (prevTdCamera !== tdCamera) { actions = actions.concat(updateTdCamera()); } + */ return actions; } diff --git a/frontend/javascripts/oxalis/model/sagas/update_actions.ts b/frontend/javascripts/oxalis/model/sagas/update_actions.ts index 702a918260a..5f95cd263ad 100644 --- a/frontend/javascripts/oxalis/model/sagas/update_actions.ts +++ b/frontend/javascripts/oxalis/model/sagas/update_actions.ts @@ -420,7 +420,7 @@ export function removeFallbackLayer() { } export function updateTdCamera() { return { - name: "updateTdCameraSkeleton", + name: "updateTdCamera", value: {}, } as const; } diff --git a/frontend/javascripts/test/helpers/saveHelpers.ts b/frontend/javascripts/test/helpers/saveHelpers.ts index 53ba1f35865..2672cb89f34 100644 --- a/frontend/javascripts/test/helpers/saveHelpers.ts +++ b/frontend/javascripts/test/helpers/saveHelpers.ts @@ -21,7 +21,9 @@ export function createSaveQueueFromUpdateActions( })); } export function withoutUpdateTracing(items: Array): Array { - return items.filter((item) => item.name !== "updateTracing"); + return items.filter( + (item) => item.name !== "updateSkeletonTracing" && item.name !== "updateVolumeTracing", + ); } export function withoutUpdateTree(items: Array): Array { return items.filter((item) => item.name !== "updateTree"); diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala index 3e0efeac89a..0738dd3c8c3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala @@ -181,12 +181,11 @@ class AnnotationTransactionService @Inject()( } } yield newVersion - def handleUpdateGroup(annotationId: String, updateActionGroup: UpdateActionGroup, userToken: Option[String])( + private def handleUpdateGroup(annotationId: String, updateActionGroup: UpdateActionGroup, userToken: Option[String])( implicit ec: ExecutionContext): Fox[Unit] = for { - _ <- tracingDataStore.annotationUpdates.put(annotationId, - updateActionGroup.version, - preprocessActionsForStorage(updateActionGroup)) + updateActionsJson <- Fox.successful(Json.toJson(preprocessActionsForStorage(updateActionGroup))) + _ <- tracingDataStore.annotationUpdates.put(annotationId, updateActionGroup.version, updateActionsJson) bucketMutatingActions = findBucketMutatingActions(updateActionGroup) _ <- Fox.runIf(bucketMutatingActions.nonEmpty)( volumeTracingService @@ -207,7 +206,7 @@ class AnnotationTransactionService @Inject()( case first :: rest => first.addInfo(updateActionGroup.info) :: rest } actionsWithInfo.map { - case a: UpdateBucketVolumeAction => a.transformToCompact // TODO or not? + case a: UpdateBucketVolumeAction => a.withoutBase64Data case a => a } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index f41824d1dfa..eb46397ca28 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -19,6 +19,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{ } import com.scalableminds.webknossos.tracingstore.tracings.volume.{ ApplyableVolumeUpdateAction, + BucketMutatingVolumeUpdateAction, VolumeTracingService, VolumeUpdateAction } @@ -85,9 +86,11 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl annotationWithTracings.applyVolumeAction(a) case a: EditableMappingUpdateAction => annotationWithTracings.applyEditableMappingAction(a) + case _: BucketMutatingVolumeUpdateAction => + Fox.successful(annotationWithTracings) // No-op, as bucket-mutating actions are performed eagerly, so not here. // TODO make Mapping Editable // Note: UpdateBucketVolumeActions are not handled here, but instead eagerly on saving. - case _ => Fox.failure("Received unsupported AnnotationUpdateAction action") + case _ => Fox.failure(s"Received unsupported AnnotationUpdateAction action ${Json.toJson(updateAction)}") } } yield updated diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala index d0e00c49819..8c7d0e34cb8 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala @@ -198,6 +198,8 @@ object UpdateAction { case s: UpdateSegmentGroupsVolumeAction => Json.obj("name" -> "updateSegmentGroups", "value" -> Json.toJson(s)(UpdateSegmentGroupsVolumeAction.jsonFormat)) case s: CompactVolumeUpdateAction => Json.toJson(s)(CompactVolumeUpdateAction.compactVolumeUpdateActionFormat) + case s: UpdateMappingNameVolumeAction => + Json.obj("name" -> "updateMappingName", "value" -> Json.toJson(s)(UpdateMappingNameVolumeAction.jsonFormat)) // Editable Mapping case s: SplitAgglomerateUpdateAction => diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index b22c0acc604..b20bf1525ff 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -45,6 +45,7 @@ import play.api.libs.json.{JsObject, JsValue, Json} import java.io._ import java.nio.file.Paths +import java.util.Base64 import java.util.zip.Deflater import scala.collection.mutable import scala.concurrent.ExecutionContext @@ -175,7 +176,8 @@ class VolumeTracingService @Inject()( action.additionalCoordinates) _ <- bool2Fox(!bucketPosition.hasNegativeComponent) ?~> s"Received a bucket at negative position ($bucketPosition), must be positive" dataLayer = volumeTracingLayer(tracingId, volumeTracing) - _ <- saveBucket(dataLayer, bucketPosition, action.data, updateGroupVersion) ?~> "failed to save bucket" + actionBucketData <- action.base64Data.map(Base64.getDecoder.decode).toFox + _ <- saveBucket(dataLayer, bucketPosition, actionBucketData, updateGroupVersion) ?~> "failed to save bucket" mappingName <- baseMappingName(volumeTracing) _ <- Fox.runIfOptionTrue(volumeTracing.hasSegmentIndex) { for { @@ -183,7 +185,7 @@ class VolumeTracingService @Inject()( _ <- updateSegmentIndex( segmentIndexBuffer, bucketPosition, - action.data, + actionBucketData, previousBucketBytes, volumeTracing.elementClass, mappingName, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala index 303c8e0969e..d00a7707d10 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala @@ -37,22 +37,21 @@ trait BucketMutatingVolumeUpdateAction extends VolumeUpdateAction case class UpdateBucketVolumeAction(position: Vec3Int, cubeSize: Int, mag: Vec3Int, - base64Data: String, + base64Data: Option[String], additionalCoordinates: Option[Seq[AdditionalCoordinate]] = None, actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) extends BucketMutatingVolumeUpdateAction { - lazy val data: Array[Byte] = Base64.getDecoder.decode(base64Data) override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - def transformToCompact: CompactVolumeUpdateAction = - CompactVolumeUpdateAction("updateBucket", Json.obj(), actionTracingId, actionTimestamp, actionAuthorId, info) + def withoutBase64Data: UpdateBucketVolumeAction = + this.copy(base64Data = None) } case class UpdateTracingVolumeAction( From fee3368c6c8c48e2fd8c2c1d303256007280ae14 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Mon, 16 Sep 2024 13:24:33 +0200 Subject: [PATCH 048/150] fix typing --- .../compaction/compact_toggle_actions.ts | 1 + .../compaction/compact_update_actions.ts | 1 + .../oxalis/model/reducers/save_reducer.ts | 50 +++++++++------ .../oxalis/model/sagas/save_saga.ts | 14 ++--- .../javascripts/oxalis/view/version_entry.tsx | 9 +-- .../backend-snapshot-tests/annotations.e2e.ts | 7 ++- .../javascripts/test/helpers/saveHelpers.ts | 4 +- .../test/reducers/save_reducer.spec.ts | 18 +++--- .../test/sagas/saga_integration.spec.ts | 1 + .../javascripts/test/sagas/save_saga.spec.ts | 14 ++++- .../test/sagas/skeletontracing_saga.spec.ts | 62 ++++++++++++++++--- 11 files changed, 129 insertions(+), 52 deletions(-) diff --git a/frontend/javascripts/oxalis/model/helpers/compaction/compact_toggle_actions.ts b/frontend/javascripts/oxalis/model/helpers/compaction/compact_toggle_actions.ts index cc7ee5af199..fd989003fab 100644 --- a/frontend/javascripts/oxalis/model/helpers/compaction/compact_toggle_actions.ts +++ b/frontend/javascripts/oxalis/model/helpers/compaction/compact_toggle_actions.ts @@ -7,6 +7,7 @@ import _ from "lodash"; import type { SkeletonTracing, Tree, TreeGroup, TreeMap, VolumeTracing } from "oxalis/store"; import type { UpdateAction, + UpdateActionWithTracingId, UpdateTreeVisibilityUpdateAction, } from "oxalis/model/sagas/update_actions"; import { updateTreeGroupVisibility, updateTreeVisibility } from "oxalis/model/sagas/update_actions"; diff --git a/frontend/javascripts/oxalis/model/helpers/compaction/compact_update_actions.ts b/frontend/javascripts/oxalis/model/helpers/compaction/compact_update_actions.ts index b16e490e5e8..c9e714a8229 100644 --- a/frontend/javascripts/oxalis/model/helpers/compaction/compact_update_actions.ts +++ b/frontend/javascripts/oxalis/model/helpers/compaction/compact_update_actions.ts @@ -7,6 +7,7 @@ import type { DeleteNodeUpdateAction, DeleteTreeUpdateAction, UpdateAction, + UpdateActionWithTracingId, } from "oxalis/model/sagas/update_actions"; import { moveTreeComponent } from "oxalis/model/sagas/update_actions"; import compactToggleActions from "oxalis/model/helpers/compaction/compact_toggle_actions"; diff --git a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts index a1d1ccf668e..13d9e7c38f3 100644 --- a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts +++ b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts @@ -18,7 +18,7 @@ import { } from "oxalis/model/reducers/volumetracing_reducer_helpers"; import Date from "libs/date"; import * as Utils from "libs/utils"; -import type { UpdateActionWithTracingId } from "../sagas/update_actions"; +import type { UpdateAction, UpdateActionWithTracingId } from "../sagas/update_actions"; // These update actions are not idempotent. Having them // twice in the save queue causes a corruption of the current annotation. @@ -135,16 +135,19 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState { } case "PUSH_SAVE_QUEUE_TRANSACTION": { - const { items, transactionId } = action; + // Use `dispatchedAction` to better distinguish this variable from + // update actions. + const dispatchedAction = action; + const { items, transactionId } = dispatchedAction; if (items.length === 0) { return state; } // Only report tracing statistics, if a "real" update to the tracing happened const stats = _.some( - action.items, + dispatchedAction.items, (ua) => ua.name !== "updateSkeletonTracing" && ua.name !== "updateVolumeTracing", ) - ? getStats(state.tracing, action.saveQueueType, action.tracingId) + ? getStats(state.tracing, dispatchedAction.saveQueueType, dispatchedAction.tracingId) : null; const { activeUser } = state; if (activeUser == null) { @@ -153,12 +156,16 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState { const updateActionChunks = _.chunk( items, - MAXIMUM_ACTION_COUNT_PER_BATCH[action.saveQueueType], + MAXIMUM_ACTION_COUNT_PER_BATCH[dispatchedAction.saveQueueType], ); const transactionGroupCount = updateActionChunks.length; const actionLogInfo = JSON.stringify(getActionLog().slice(-10)); - const oldQueue = selectQueue(state, action.saveQueueType, action.tracingId); + const oldQueue = selectQueue( + state, + dispatchedAction.saveQueueType, + dispatchedAction.tracingId, + ); const newQueue = oldQueue.concat( updateActionChunks.map((actions, transactionGroupIndex) => ({ // Placeholder, the version number will be updated before sending to the server @@ -168,16 +175,7 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState { transactionGroupIndex, timestamp: Date.now(), authorId: activeUser.id, - actions: actions.map( - (innerAction) => - ({ - ...innerAction, - value: { - ...innerAction.value, - actionTracingId: action.tracingId, - }, - }) as UpdateActionWithTracingId, - ), + actions: addTracingIdToActions(actions, dispatchedAction.tracingId), stats, // Redux Action Log context for debugging purposes. info: actionLogInfo, @@ -189,7 +187,7 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState { // caught by the following check. If the bug appears again, we can investigate with more // details thanks to airbrake. if ( - action.saveQueueType === "skeleton" && + dispatchedAction.saveQueueType === "skeleton" && oldQueue.length > 0 && newQueue.length > 0 && newQueue.at(-1)?.actions.some((action) => NOT_IDEMPOTENT_ACTIONS.includes(action.name)) && @@ -205,7 +203,7 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState { ); } - const newQueueObj = updateTracingDict(action, state.save.queue, newQueue); + const newQueueObj = updateTracingDict(dispatchedAction, state.save.queue, newQueue); return update(state, { save: { queue: { @@ -314,4 +312,20 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState { } } +export function addTracingIdToActions( + actions: UpdateAction[], + tracingId: string, +): UpdateActionWithTracingId[] { + return actions.map( + (innerAction) => + ({ + ...innerAction, + value: { + ...innerAction.value, + actionTracingId: tracingId, + }, + }) as UpdateActionWithTracingId, + ); +} + export default SaveReducer; diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts index b51dd0152dc..e12165012f1 100644 --- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts @@ -105,7 +105,7 @@ export function* pushSaveQueueAsync(saveQueueType: SaveQueueType, tracingId: str // ignored (they will be picked up in the next iteration of this loop). // Otherwise, the risk of a high number of save-requests (see case 1) // would be present here, too (note the risk would be greater, because the - // user didn't use the save button which is usually accompanied a small pause). + // user didn't use the save button which is usually accompanied by a small pause). const itemCountToSave = forcePush ? Number.POSITIVE_INFINITY : yield* select((state) => selectQueue(state, saveQueueType, tracingId).length); @@ -175,9 +175,7 @@ export function* sendRequestToServer( const fullSaveQueue = yield* select((state) => selectQueue(state, saveQueueType, tracingId)); const saveQueue = sliceAppropriateBatchCount(fullSaveQueue, saveQueueType); let compactedSaveQueue = compactSaveQueue(saveQueue); - const { version, type } = yield* select((state) => - selectTracing(state, saveQueueType, tracingId), - ); + const { version } = yield* select((state) => selectTracing(state, saveQueueType, tracingId)); const annotationId = yield* select((state) => state.tracing.annotationId); const tracingStoreUrl = yield* select((state) => state.tracing.tracingStore.url); let versionIncrement; @@ -354,8 +352,8 @@ export function performDiffTracing( tracing: SkeletonTracing | VolumeTracing, prevFlycam: Flycam, flycam: Flycam, - prevTdCamera: CameraData, - tdCamera: CameraData, + _prevTdCamera: CameraData, + _tdCamera: CameraData, ): Array { let actions: Array = []; @@ -372,7 +370,7 @@ export function performDiffTracing( } /* - TODO: restore this update action (decide how to handle it, does it belong to skeleton or volume or something else?) + TODOp: restore this update action (decide how to handle it, does it belong to skeleton or volume or something else?) if (prevTdCamera !== tdCamera) { actions = actions.concat(updateTdCamera()); } @@ -401,7 +399,7 @@ export function* setupSavingForTracingType( /* Listen to changes to the annotation and derive UpdateActions from the old and new state. - The actual push to the server is done by the forked pushSaveQueueAsync saga. + The actual push to the server is done by the forked pushSaveQueueAsync saga. */ const saveQueueType = initializeAction.type === "INITIALIZE_SKELETONTRACING" ? "skeleton" : "volume"; diff --git a/frontend/javascripts/oxalis/view/version_entry.tsx b/frontend/javascripts/oxalis/view/version_entry.tsx index c792f7171d6..4302438ded2 100644 --- a/frontend/javascripts/oxalis/view/version_entry.tsx +++ b/frontend/javascripts/oxalis/view/version_entry.tsx @@ -188,8 +188,9 @@ const descriptionFns: Record Descr description: `Merged the trees with id ${action.value.sourceId} and ${action.value.targetId}.`, icon: , }), - updateTracing: (): Description => updateTracingDescription, -}; + updateSkeletonTracing: (): Description => updateTracingDescription, + updateVolumeTracing: (): Description => updateTracingDescription, +} as const; function getDescriptionForSpecificBatch( actions: Array, @@ -200,8 +201,8 @@ function getDescriptionForSpecificBatch( if (firstAction.name !== type) { throw new Error("Type constraint violated"); } - - return descriptionFns[type](firstAction, actions.length); + const fn = descriptionFns[type]; + return fn(firstAction, actions.length); } // An update action batch can consist of more than one update action as a single user action diff --git a/frontend/javascripts/test/backend-snapshot-tests/annotations.e2e.ts b/frontend/javascripts/test/backend-snapshot-tests/annotations.e2e.ts index b698c52ae54..780739fc832 100644 --- a/frontend/javascripts/test/backend-snapshot-tests/annotations.e2e.ts +++ b/frontend/javascripts/test/backend-snapshot-tests/annotations.e2e.ts @@ -20,6 +20,8 @@ import * as api from "admin/admin_rest_api"; import generateDummyTrees from "oxalis/model/helpers/generate_dummy_trees"; import test from "ava"; import { createSaveQueueFromUpdateActions } from "../helpers/saveHelpers"; +import type { SaveQueueEntry } from "oxalis/store"; + const datasetId = { name: "confocal-multi_knossos", owningOrganization: "Organization_X", @@ -146,8 +148,7 @@ test.serial("getTracingsForAnnotation() for hybrid", async (t) => { }); }); -// @ts-expect-error ts-migrate(7006) FIXME: Parameter 'queue' implicitly has an 'any' type. -async function sendUpdateActionsForSkeleton(explorational: APIAnnotation, queue) { +async function sendUpdateActionsForSkeleton(explorational: APIAnnotation, queue: SaveQueueEntry[]) { const skeletonTracing = getSkeletonDescriptor(explorational); if (skeletonTracing == null) throw new Error("No skeleton annotation present."); return sendRequestWithToken( @@ -173,6 +174,7 @@ test.serial("Send update actions and compare resulting tracing", async (t) => { [UpdateActions.updateSkeletonTracing(initialSkeleton, [2, 3, 4], null, [1, 2, 3], 2)], ], 123456789, + createdExplorational.annotationLayers[0].tracingId, ), 0, ); @@ -207,6 +209,7 @@ test("Send complex update actions and compare resulting tracing", async (t) => { createSaveQueueFromUpdateActions( [createTreesUpdateActions, [updateTreeGroupsUpdateAction]], 123456789, + createdExplorational.annotationLayers[0].tracingId, ), 0, ); diff --git a/frontend/javascripts/test/helpers/saveHelpers.ts b/frontend/javascripts/test/helpers/saveHelpers.ts index 2672cb89f34..09703d25e29 100644 --- a/frontend/javascripts/test/helpers/saveHelpers.ts +++ b/frontend/javascripts/test/helpers/saveHelpers.ts @@ -1,4 +1,5 @@ import type { TracingStats } from "oxalis/model/accessors/annotation_accessor"; +import { addTracingIdToActions } from "oxalis/model/reducers/save_reducer"; import type { UpdateAction } from "oxalis/model/sagas/update_actions"; import type { SaveQueueEntry } from "oxalis/store"; import dummyUser from "test/fixtures/dummy_user"; @@ -6,13 +7,14 @@ import dummyUser from "test/fixtures/dummy_user"; export function createSaveQueueFromUpdateActions( updateActions: UpdateAction[][], timestamp: number, + tracingId: string, stats: TracingStats | null = null, ): SaveQueueEntry[] { return updateActions.map((ua) => ({ version: -1, timestamp, stats, - actions: ua.slice(), + actions: addTracingIdToActions(ua, tracingId), info: "[]", transactionGroupCount: 1, authorId: dummyUser.id, diff --git a/frontend/javascripts/test/reducers/save_reducer.spec.ts b/frontend/javascripts/test/reducers/save_reducer.spec.ts index 0a4b398f274..1ae173ee944 100644 --- a/frontend/javascripts/test/reducers/save_reducer.spec.ts +++ b/frontend/javascripts/test/reducers/save_reducer.spec.ts @@ -6,6 +6,7 @@ import type { SaveState } from "oxalis/store"; import type { APIUser } from "types/api_flow_types"; import { createSaveQueueFromUpdateActions } from "../helpers/saveHelpers"; import type { EmptyObject } from "types/globals"; + const TIMESTAMP = 1494695001688; const DateMock = { now: () => TIMESTAMP, @@ -19,6 +20,7 @@ const SaveActions = mockRequire.reRequire("oxalis/model/actions/save_actions"); const SaveReducer = mockRequire.reRequire("oxalis/model/reducers/save_reducer").default; const { createEdge } = mockRequire.reRequire("oxalis/model/sagas/update_actions"); +const tracingId = "1234567890"; const initialState: { save: SaveState; activeUser: APIUser; tracing: EmptyObject } = { activeUser: dummyUser, save: { @@ -46,14 +48,18 @@ const initialState: { save: SaveState; activeUser: APIUser; tracing: EmptyObject }; test("Save should add update actions to the queue", (t) => { const items = [createEdge(0, 1, 2), createEdge(0, 2, 3)]; - const saveQueue = createSaveQueueFromUpdateActions([items], TIMESTAMP); + const saveQueue = createSaveQueueFromUpdateActions([items], TIMESTAMP, tracingId); const pushAction = SaveActions.pushSaveQueueTransaction(items, "skeleton"); const newState = SaveReducer(initialState, pushAction); t.deepEqual(newState.save.queue.skeleton, saveQueue); }); test("Save should add more update actions to the queue", (t) => { const getItems = (treeId: number) => [createEdge(treeId, 1, 2), createEdge(treeId, 2, 3)]; - const saveQueue = createSaveQueueFromUpdateActions([getItems(0), getItems(1)], TIMESTAMP); + const saveQueue = createSaveQueueFromUpdateActions( + [getItems(0), getItems(1)], + TIMESTAMP, + tracingId, + ); const testState = SaveReducer( initialState, SaveActions.pushSaveQueueTransaction(getItems(0), "skeleton"), @@ -65,9 +71,7 @@ test("Save should add more update actions to the queue", (t) => { t.deepEqual(newState.save.queue.skeleton, saveQueue); }); test("Save should add zero update actions to the queue", (t) => { - // @ts-expect-error ts-migrate(7034) FIXME: Variable 'items' implicitly has type 'any[]' in so... Remove this comment to see the full error message - const items = []; - // @ts-expect-error ts-migrate(7005) FIXME: Variable 'items' implicitly has an 'any[]' type. + const items = [] as const; const pushAction = SaveActions.pushSaveQueueTransaction(items, "skeleton"); const newState = SaveReducer(initialState, pushAction); t.deepEqual(newState.save.queue.skeleton, []); @@ -75,7 +79,7 @@ test("Save should add zero update actions to the queue", (t) => { test("Save should remove one update actions from the queue", (t) => { const firstItem = [createEdge(0, 1, 2)]; const secondItem = [createEdge(1, 2, 3)]; - const saveQueue = createSaveQueueFromUpdateActions([secondItem], TIMESTAMP); + const saveQueue = createSaveQueueFromUpdateActions([secondItem], TIMESTAMP, tracingId); const firstPushAction = SaveActions.pushSaveQueueTransaction(firstItem, "skeleton"); const secondPushAction = SaveActions.pushSaveQueueTransaction(secondItem, "skeleton"); const popAction = SaveActions.shiftSaveQueueAction(1, "skeleton"); @@ -86,7 +90,7 @@ test("Save should remove one update actions from the queue", (t) => { }); test("Save should remove zero update actions from the queue", (t) => { const items = [createEdge(0, 1, 2), createEdge(1, 2, 3)]; - const saveQueue = createSaveQueueFromUpdateActions([items], TIMESTAMP); + const saveQueue = createSaveQueueFromUpdateActions([items], TIMESTAMP, tracingId); const pushAction = SaveActions.pushSaveQueueTransaction(items, "skeleton"); const popAction = SaveActions.shiftSaveQueueAction(0, "skeleton"); let newState = SaveReducer(initialState, pushAction); diff --git a/frontend/javascripts/test/sagas/saga_integration.spec.ts b/frontend/javascripts/test/sagas/saga_integration.spec.ts index 771fb88b194..c74623c26d9 100644 --- a/frontend/javascripts/test/sagas/saga_integration.spec.ts +++ b/frontend/javascripts/test/sagas/saga_integration.spec.ts @@ -67,6 +67,7 @@ test.serial( ], ], TIMESTAMP, + "tracingId", getStats(state.tracing, "skeleton", "irrelevant_in_skeleton_case") || undefined, ); // Reset the info field which is just for debugging purposes diff --git a/frontend/javascripts/test/sagas/save_saga.spec.ts b/frontend/javascripts/test/sagas/save_saga.spec.ts index 677568ef279..ed9f3146fa8 100644 --- a/frontend/javascripts/test/sagas/save_saga.spec.ts +++ b/frontend/javascripts/test/sagas/save_saga.spec.ts @@ -75,12 +75,13 @@ test("SaveSaga should compact multiple updateTracing update actions", (t) => { [UpdateActions.updateSkeletonTracing(initialState, [2, 3, 4], [0, 0, 1], 2)], ], TIMESTAMP, + tracingId, ); t.deepEqual(compactSaveQueue(saveQueue), [saveQueue[1]]); }); test("SaveSaga should send update actions", (t) => { const updateActions = [[UpdateActions.createEdge(1, 0, 1)], [UpdateActions.createEdge(1, 1, 2)]]; - const saveQueue = createSaveQueueFromUpdateActions(updateActions, TIMESTAMP); + const saveQueue = createSaveQueueFromUpdateActions(updateActions, TIMESTAMP, tracingId); const saga = pushSaveQueueAsync(TRACING_TYPE, tracingId); expectValueDeepEqual(t, saga.next(), call(ensureWkReady)); saga.next(); // setLastSaveTimestampAction @@ -113,6 +114,7 @@ test("SaveSaga should send request to server", (t) => { const saveQueue = createSaveQueueFromUpdateActions( [[UpdateActions.createEdge(1, 0, 1)], [UpdateActions.createEdge(1, 1, 2)]], TIMESTAMP, + tracingId, ); const saga = sendRequestToServer(TRACING_TYPE, tracingId); saga.next(); @@ -137,6 +139,7 @@ test("SaveSaga should retry update actions", (t) => { const saveQueue = createSaveQueueFromUpdateActions( [[UpdateActions.createEdge(1, 0, 1)], [UpdateActions.createEdge(1, 1, 2)]], TIMESTAMP, + tracingId, ); const [saveQueueWithVersions, versionIncrement] = addVersionNumbers(saveQueue, LAST_VERSION); t.is(versionIncrement, 2); @@ -170,6 +173,7 @@ test("SaveSaga should escalate on permanent client error update actions", (t) => const saveQueue = createSaveQueueFromUpdateActions( [[UpdateActions.createEdge(1, 0, 1)], [UpdateActions.createEdge(1, 1, 2)]], TIMESTAMP, + tracingId, ); const saga = sendRequestToServer(TRACING_TYPE, tracingId); saga.next(); @@ -206,7 +210,7 @@ test("SaveSaga should escalate on permanent client error update actions", (t) => }); test("SaveSaga should send update actions right away and try to reach a state where all updates are saved", (t) => { const updateActions = [[UpdateActions.createEdge(1, 0, 1)], [UpdateActions.createEdge(1, 1, 2)]]; - const saveQueue = createSaveQueueFromUpdateActions(updateActions, TIMESTAMP); + const saveQueue = createSaveQueueFromUpdateActions(updateActions, TIMESTAMP, tracingId); const saga = pushSaveQueueAsync(TRACING_TYPE, tracingId); expectValueDeepEqual(t, saga.next(), call(ensureWkReady)); saga.next(); @@ -229,7 +233,7 @@ test("SaveSaga should send update actions right away and try to reach a state wh }); test("SaveSaga should not try to reach state with all actions being saved when saving is triggered by a timeout", (t) => { const updateActions = [[UpdateActions.createEdge(1, 0, 1)], [UpdateActions.createEdge(1, 1, 2)]]; - const saveQueue = createSaveQueueFromUpdateActions(updateActions, TIMESTAMP); + const saveQueue = createSaveQueueFromUpdateActions(updateActions, TIMESTAMP, tracingId); const saga = pushSaveQueueAsync(TRACING_TYPE, tracingId); expectValueDeepEqual(t, saga.next(), call(ensureWkReady)); saga.next(); @@ -253,6 +257,7 @@ test("SaveSaga should remove the correct update actions", (t) => { [UpdateActions.updateSkeletonTracing(initialState, [2, 3, 4], [0, 0, 1], 2)], ], TIMESTAMP, + tracingId, ); const saga = sendRequestToServer(TRACING_TYPE, tracingId); saga.next(); @@ -286,6 +291,7 @@ test("SaveSaga should set the correct version numbers", (t) => { [UpdateActions.createEdge(2, 3, 4)], ], TIMESTAMP, + tracingId, ); const saga = sendRequestToServer(TRACING_TYPE, tracingId); saga.next(); @@ -319,6 +325,7 @@ test("SaveSaga should set the correct version numbers if the save queue was comp [UpdateActions.updateSkeletonTracing(initialState, [3, 4, 5], [0, 0, 1], 3)], ], TIMESTAMP, + tracingId, ); const saga = sendRequestToServer(TRACING_TYPE, tracingId); saga.next(); @@ -354,6 +361,7 @@ test("SaveSaga addVersionNumbers should set the correct version numbers", (t) => ], TIMESTAMP, + tracingId, ); const [saveQueueWithVersions, versionIncrement] = addVersionNumbers(saveQueue, LAST_VERSION); t.is(versionIncrement, 3); diff --git a/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts b/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts index 619047cdaca..f612805bcc7 100644 --- a/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts +++ b/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts @@ -27,6 +27,7 @@ import { TreeTypeEnum } from "oxalis/constants"; import type { Action } from "oxalis/model/actions/actions"; import type { ServerSkeletonTracing } from "types/api_flow_types"; import { enforceSkeletonTracing } from "oxalis/model/accessors/skeletontracing_accessor"; +import { addTracingIdToActions } from "oxalis/model/reducers/save_reducer"; const TIMESTAMP = 1494347146379; const DateMock = { @@ -82,6 +83,13 @@ function compactSaveQueueWithUpdateActions( tracing: SkeletonTracing, ): Array { return compactSaveQueue( + // todop + // Do we really need compactSaveQueueWithUpdateActions? actually, compactUpdateActions + // is never called with a save queue in prod (instead, the function is called before + // filling the save queue). one could probably combine compactUpdateActions and + // createSaveQueueFromUpdateActions to have a createCompactedSaveQueueFromUpdateActions + // helper function and use that in this spec. + // @ts-expect-error queue.map((batch) => ({ ...batch, actions: compactUpdateActions(batch.actions, tracing) })), ); } @@ -634,7 +642,11 @@ test("compactUpdateActions should detect a tree merge (1/3)", (t) => { testState.flycam, newState.flycam, ); - const saveQueue = createSaveQueueFromUpdateActions([updateActions], TIMESTAMP); + const saveQueue = createSaveQueueFromUpdateActions( + [updateActions], + TIMESTAMP, + skeletonTracing.tracingId, + ); const simplifiedUpdateActions = compactSaveQueueWithUpdateActions( saveQueue, enforceSkeletonTracing(newState.tracing), @@ -694,7 +706,11 @@ test("compactUpdateActions should detect a tree merge (2/3)", (t) => { testDiffing(newState1.tracing, newState2.tracing, newState1.flycam, newState2.flycam), ); // compactUpdateActions is triggered by the saving, it can therefore contain the results of more than one diffing - const saveQueue = createSaveQueueFromUpdateActions(updateActions, TIMESTAMP); + const saveQueue = createSaveQueueFromUpdateActions( + updateActions, + TIMESTAMP, + skeletonTracing.tracingId, + ); const simplifiedUpdateActions = compactSaveQueueWithUpdateActions( saveQueue, enforceSkeletonTracing(newState2.tracing), @@ -796,7 +812,11 @@ test("compactUpdateActions should detect a tree merge (3/3)", (t) => { ), ); // compactUpdateActions is triggered by the saving, it can therefore contain the results of more than one diffing - const saveQueue = createSaveQueueFromUpdateActions(updateActions, TIMESTAMP); + const saveQueue = createSaveQueueFromUpdateActions( + updateActions, + TIMESTAMP, + skeletonTracing.tracingId, + ); const simplifiedUpdateActions = compactSaveQueueWithUpdateActions( saveQueue, enforceSkeletonTracing(newState.tracing), @@ -878,7 +898,11 @@ test("compactUpdateActions should detect a tree split (1/3)", (t) => { testState.flycam, newState.flycam, ); - const saveQueue = createSaveQueueFromUpdateActions([updateActions], TIMESTAMP); + const saveQueue = createSaveQueueFromUpdateActions( + [updateActions], + TIMESTAMP, + skeletonTracing.tracingId, + ); const simplifiedUpdateActions = compactSaveQueueWithUpdateActions( saveQueue, enforceSkeletonTracing(newState.tracing), @@ -936,7 +960,11 @@ test("compactUpdateActions should detect a tree split (2/3)", (t) => { testState.flycam, newState.flycam, ); - const saveQueue = createSaveQueueFromUpdateActions([updateActions], TIMESTAMP); + const saveQueue = createSaveQueueFromUpdateActions( + [updateActions], + TIMESTAMP, + skeletonTracing.tracingId, + ); const simplifiedUpdateActions = compactSaveQueueWithUpdateActions( saveQueue, enforceSkeletonTracing(newState.tracing), @@ -1008,7 +1036,11 @@ test("compactUpdateActions should detect a tree split (3/3)", (t) => { updateActions.push( testDiffing(newState1.tracing, newState2.tracing, newState1.flycam, newState2.flycam), ); - const saveQueue = createSaveQueueFromUpdateActions(updateActions, TIMESTAMP); + const saveQueue = createSaveQueueFromUpdateActions( + updateActions, + TIMESTAMP, + skeletonTracing.tracingId, + ); const simplifiedUpdateActions = compactSaveQueueWithUpdateActions( saveQueue, enforceSkeletonTracing(newState2.tracing), @@ -1095,7 +1127,11 @@ test("compactUpdateActions should do nothing if it cannot compact", (t) => { testState.flycam, newState.flycam, ); - const saveQueue = createSaveQueueFromUpdateActions([updateActions], TIMESTAMP); + const saveQueue = createSaveQueueFromUpdateActions( + [updateActions], + TIMESTAMP, + skeletonTracing.tracingId, + ); const simplifiedUpdateActions = compactSaveQueueWithUpdateActions( saveQueue, enforceSkeletonTracing(newState.tracing), @@ -1124,7 +1160,11 @@ test("compactUpdateActions should detect a deleted tree", (t) => { testState.flycam, newState.flycam, ); - const saveQueue = createSaveQueueFromUpdateActions([updateActions], TIMESTAMP); + const saveQueue = createSaveQueueFromUpdateActions( + [updateActions], + TIMESTAMP, + skeletonTracing.tracingId, + ); const simplifiedUpdateActions = compactSaveQueueWithUpdateActions( saveQueue, enforceSkeletonTracing(newState.tracing), @@ -1156,7 +1196,11 @@ test("compactUpdateActions should not detect a deleted tree if there is no delet testState.flycam, newState.flycam, ); - const saveQueue = createSaveQueueFromUpdateActions([updateActions], TIMESTAMP); + const saveQueue = createSaveQueueFromUpdateActions( + [updateActions], + TIMESTAMP, + skeletonTracing.tracingId, + ); const simplifiedUpdateActions = compactSaveQueueWithUpdateActions( saveQueue, enforceSkeletonTracing(newState.tracing), From 4e54e85787f79e3275461fb8c375440bd9ebaff3 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Mon, 16 Sep 2024 13:42:00 +0200 Subject: [PATCH 049/150] only have one save queue for all tracings in an annotation --- frontend/javascripts/oxalis/default_state.ts | 6 +-- .../oxalis/model/accessors/save_accessor.ts | 16 ------ .../oxalis/model/reducers/save_reducer.ts | 49 +++---------------- .../oxalis/model/sagas/save_saga.ts | 13 ++--- frontend/javascripts/oxalis/store.ts | 6 +-- .../oxalis/view/action-bar/save_button.tsx | 20 +++----- .../test/reducers/save_reducer.spec.ts | 6 +-- .../test/sagas/saga_integration.spec.ts | 21 ++++---- 8 files changed, 32 insertions(+), 105 deletions(-) diff --git a/frontend/javascripts/oxalis/default_state.ts b/frontend/javascripts/oxalis/default_state.ts index 298aa359a17..090a3384aa6 100644 --- a/frontend/javascripts/oxalis/default_state.ts +++ b/frontend/javascripts/oxalis/default_state.ts @@ -179,11 +179,7 @@ const defaultState: OxalisState = { annotationLayers: [], }, save: { - queue: { - skeleton: [], - volumes: {}, - mappings: {}, - }, + queue: [], isBusyInfo: { skeleton: false, volumes: {}, diff --git a/frontend/javascripts/oxalis/model/accessors/save_accessor.ts b/frontend/javascripts/oxalis/model/accessors/save_accessor.ts index e35ed69805d..d9dd9d65aa0 100644 --- a/frontend/javascripts/oxalis/model/accessors/save_accessor.ts +++ b/frontend/javascripts/oxalis/model/accessors/save_accessor.ts @@ -9,19 +9,3 @@ export function isBusy(isBusyInfo: IsBusyInfo): boolean { Utils.values(isBusyInfo.mappings).some((el) => el) ); } -export function selectQueue( - state: OxalisState, - saveQueueType: SaveQueueType, - tracingId: string, -): Array { - switch (saveQueueType) { - case "skeleton": - return state.save.queue.skeleton; - case "volume": - return state.save.queue.volumes[tracingId]; - case "mapping": - return state.save.queue.mappings[tracingId]; - default: - throw new Error(`Unknown save queue type: ${saveQueueType}`); - } -} diff --git a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts index 13d9e7c38f3..b8d75ff86dc 100644 --- a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts +++ b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts @@ -10,7 +10,6 @@ import type { import { getActionLog } from "oxalis/model/helpers/action_logger_middleware"; import { getStats } from "oxalis/model/accessors/annotation_accessor"; import { MAXIMUM_ACTION_COUNT_PER_BATCH } from "oxalis/model/sagas/save_saga_constants"; -import { selectQueue } from "oxalis/model/accessors/save_accessor"; import { updateKey2 } from "oxalis/model/helpers/deep_update"; import { updateEditableMapping, @@ -61,15 +60,7 @@ function updateTracingDict( } export function getTotalSaveQueueLength(queueObj: SaveState["queue"]) { - return ( - queueObj.skeleton.length + - _.sum( - Utils.values(queueObj.volumes).map((volumeQueue: SaveQueueEntry[]) => volumeQueue.length), - ) + - _.sum( - Utils.values(queueObj.mappings).map((mappingQueue: SaveQueueEntry[]) => mappingQueue.length), - ) - ); + return queueObj.length; } function updateVersion(state: OxalisState, action: SetVersionNumberAction) { @@ -118,22 +109,6 @@ function updateLastSaveTimestamp(state: OxalisState, action: SetLastSaveTimestam function SaveReducer(state: OxalisState, action: Action): OxalisState { switch (action.type) { - case "INITIALIZE_VOLUMETRACING": { - // Set up empty save queue array for volume tracing - const newVolumesQueue = { ...state.save.queue.volumes, [action.tracing.id]: [] }; - return updateKey2(state, "save", "queue", { - volumes: newVolumesQueue, - }); - } - - case "INITIALIZE_EDITABLE_MAPPING": { - // Set up empty save queue array for editable mapping - const newMappingsQueue = { ...state.save.queue.mappings, [action.mapping.tracingId]: [] }; - return updateKey2(state, "save", "queue", { - mappings: newMappingsQueue, - }); - } - case "PUSH_SAVE_QUEUE_TRANSACTION": { // Use `dispatchedAction` to better distinguish this variable from // update actions. @@ -161,11 +136,7 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState { const transactionGroupCount = updateActionChunks.length; const actionLogInfo = JSON.stringify(getActionLog().slice(-10)); - const oldQueue = selectQueue( - state, - dispatchedAction.saveQueueType, - dispatchedAction.tracingId, - ); + const oldQueue = state.save.queue; const newQueue = oldQueue.concat( updateActionChunks.map((actions, transactionGroupIndex) => ({ // Placeholder, the version number will be updated before sending to the server @@ -203,11 +174,10 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState { ); } - const newQueueObj = updateTracingDict(dispatchedAction, state.save.queue, newQueue); return update(state, { save: { queue: { - $set: newQueueObj, + $set: newQueue, }, progressInfo: { totalActionCount: { @@ -222,7 +192,7 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState { const { count } = action; if (count > 0) { - const queue = selectQueue(state, action.saveQueueType, action.tracingId); + const queue = state.save.queue; const processedQueueActionCount = _.sumBy( queue.slice(0, count), @@ -230,13 +200,12 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState { ); const remainingQueue = queue.slice(count); - const newQueueObj = updateTracingDict(action, state.save.queue, remainingQueue); - const remainingQueueLength = getTotalSaveQueueLength(newQueueObj); + const remainingQueueLength = getTotalSaveQueueLength(remainingQueue); const resetCounter = remainingQueueLength === 0; return update(state, { save: { queue: { - $set: newQueueObj, + $set: remainingQueue, }, progressInfo: { // Reset progress counters if the queue is empty. Otherwise, @@ -259,11 +228,7 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState { return update(state, { save: { queue: { - $set: { - skeleton: [], - volumes: _.mapValues(state.save.queue.volumes, () => []), - mappings: _.mapValues(state.save.queue.mappings, () => []), - }, + $set: [], }, progressInfo: { processedActionCount: { diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts index e12165012f1..4cc5d6e6f5b 100644 --- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts @@ -10,7 +10,6 @@ import _ from "lodash"; import messages from "messages"; import { ControlModeEnum } from "oxalis/constants"; import { getResolutionInfo } from "oxalis/model/accessors/dataset_accessor"; -import { selectQueue } from "oxalis/model/accessors/save_accessor"; import { selectTracing } from "oxalis/model/accessors/tracing_accessor"; import { getVolumeTracingById } from "oxalis/model/accessors/volumetracing_accessor"; import { FlycamActions } from "oxalis/model/actions/flycam_actions"; @@ -69,7 +68,7 @@ export function* pushSaveQueueAsync(saveQueueType: SaveQueueType, tracingId: str let saveQueue; // Check whether the save queue is actually empty, the PUSH_SAVE_QUEUE_TRANSACTION action // could have been triggered during the call to sendRequestToServer - saveQueue = yield* select((state) => selectQueue(state, saveQueueType, tracingId)); + saveQueue = yield* select((state) => state.save.queue); if (saveQueue.length === 0) { if (loopCounter % 100 === 0) { @@ -108,10 +107,10 @@ export function* pushSaveQueueAsync(saveQueueType: SaveQueueType, tracingId: str // user didn't use the save button which is usually accompanied by a small pause). const itemCountToSave = forcePush ? Number.POSITIVE_INFINITY - : yield* select((state) => selectQueue(state, saveQueueType, tracingId).length); + : yield* select((state) => state.save.queue.length); let savedItemCount = 0; while (savedItemCount < itemCountToSave) { - saveQueue = yield* select((state) => selectQueue(state, saveQueueType, tracingId)); + saveQueue = yield* select((state) => state.save.queue); if (saveQueue.length > 0) { savedItemCount += yield* call(sendRequestToServer, saveQueueType, tracingId); @@ -172,7 +171,7 @@ export function* sendRequestToServer( * The saga returns the number of save queue items that were saved. */ - const fullSaveQueue = yield* select((state) => selectQueue(state, saveQueueType, tracingId)); + const fullSaveQueue = yield* select((state) => state.save.queue); const saveQueue = sliceAppropriateBatchCount(fullSaveQueue, saveQueueType); let compactedSaveQueue = compactSaveQueue(saveQueue); const { version } = yield* select((state) => selectTracing(state, saveQueueType, tracingId)); @@ -531,9 +530,7 @@ function* watchForSaveConflicts() { // The latest version on the server is greater than the most-recently // stored version. - const saveQueue = yield* select((state) => - selectQueue(state, tracing.type, tracing.tracingId), - ); + const saveQueue = yield* select((state) => state.save.queue); let msg = ""; if (!allowSave) { diff --git a/frontend/javascripts/oxalis/store.ts b/frontend/javascripts/oxalis/store.ts index 57ff2032def..24aca44ec1f 100644 --- a/frontend/javascripts/oxalis/store.ts +++ b/frontend/javascripts/oxalis/store.ts @@ -457,11 +457,7 @@ export type IsBusyInfo = { }; export type SaveState = { readonly isBusyInfo: IsBusyInfo; - readonly queue: { - readonly skeleton: Array; - readonly volumes: Record>; - readonly mappings: Record>; - }; + readonly queue: Array; readonly lastSaveTimestamp: { readonly skeleton: number; readonly volumes: Record; diff --git a/frontend/javascripts/oxalis/view/action-bar/save_button.tsx b/frontend/javascripts/oxalis/view/action-bar/save_button.tsx index e577c802e94..87ad0d413fb 100644 --- a/frontend/javascripts/oxalis/view/action-bar/save_button.tsx +++ b/frontend/javascripts/oxalis/view/action-bar/save_button.tsx @@ -176,18 +176,14 @@ class SaveButton extends React.PureComponent { function getOldestUnsavedTimestamp(saveQueue: SaveState["queue"]): number | null | undefined { let oldestUnsavedTimestamp; - if (saveQueue.skeleton.length > 0) { - oldestUnsavedTimestamp = saveQueue.skeleton[0].timestamp; - } - - for (const volumeQueue of Utils.values(saveQueue.volumes)) { - if (volumeQueue.length > 0) { - const oldestVolumeTimestamp = volumeQueue[0].timestamp; - oldestUnsavedTimestamp = Math.min( - oldestUnsavedTimestamp != null ? oldestUnsavedTimestamp : Number.POSITIVE_INFINITY, - oldestVolumeTimestamp, - ); - } + if (saveQueue.length > 0) { + // todop: theoretically, could this be not the oldest one? + // e.g., items are added to the queue like that: + // SkelT=1, SkelT=2, SkelT=3, VolT=1 + // now the first action is saved and the following remains: + // SkelT=2, SkelT=3, VolT=1 + // even if it could happen, probably not critical for the current context? + oldestUnsavedTimestamp = saveQueue[0].timestamp; } return oldestUnsavedTimestamp; diff --git a/frontend/javascripts/test/reducers/save_reducer.spec.ts b/frontend/javascripts/test/reducers/save_reducer.spec.ts index 1ae173ee944..b7a83326ca1 100644 --- a/frontend/javascripts/test/reducers/save_reducer.spec.ts +++ b/frontend/javascripts/test/reducers/save_reducer.spec.ts @@ -29,11 +29,7 @@ const initialState: { save: SaveState; activeUser: APIUser; tracing: EmptyObject volumes: {}, mappings: {}, }, - queue: { - skeleton: [], - volumes: {}, - mappings: {}, - }, + queue: [], lastSaveTimestamp: { skeleton: 0, volumes: {}, diff --git a/frontend/javascripts/test/sagas/saga_integration.spec.ts b/frontend/javascripts/test/sagas/saga_integration.spec.ts index c74623c26d9..33d818926c9 100644 --- a/frontend/javascripts/test/sagas/saga_integration.spec.ts +++ b/frontend/javascripts/test/sagas/saga_integration.spec.ts @@ -71,7 +71,7 @@ test.serial( getStats(state.tracing, "skeleton", "irrelevant_in_skeleton_case") || undefined, ); // Reset the info field which is just for debugging purposes - const actualSaveQueue = state.save.queue.skeleton.map((entry) => { + const actualSaveQueue = state.save.queue.map((entry) => { return { ...omit(entry, "info"), info: "[]" }; }); // Once the updateTree update action is in the save queue, we're good. @@ -82,24 +82,21 @@ test.serial( test.serial("Save actions should not be chunked below the chunk limit (1/3)", (t) => { Store.dispatch(discardSaveQueuesAction()); - t.deepEqual(Store.getState().save.queue.skeleton, []); + t.deepEqual(Store.getState().save.queue, []); const trees = generateDummyTrees(1000, 1); Store.dispatch(addTreesAndGroupsAction(createTreeMapFromTreeArray(trees), [])); - t.is(Store.getState().save.queue.skeleton.length, 1); - t.true( - Store.getState().save.queue.skeleton[0].actions.length < - MAXIMUM_ACTION_COUNT_PER_BATCH.skeleton, - ); + t.is(Store.getState().save.queue.length, 1); + t.true(Store.getState().save.queue[0].actions.length < MAXIMUM_ACTION_COUNT_PER_BATCH.skeleton); }); test.serial("Save actions should be chunked above the chunk limit (2/3)", (t) => { Store.dispatch(discardSaveQueuesAction()); - t.deepEqual(Store.getState().save.queue.skeleton, []); + t.deepEqual(Store.getState().save.queue, []); const trees = generateDummyTrees(5000, 1); Store.dispatch(addTreesAndGroupsAction(createTreeMapFromTreeArray(trees), [])); const state = Store.getState(); - t.true(state.save.queue.skeleton.length > 1); - t.is(state.save.queue.skeleton[0].actions.length, MAXIMUM_ACTION_COUNT_PER_BATCH.skeleton); + t.true(state.save.queue.length > 1); + t.is(state.save.queue[0].actions.length, MAXIMUM_ACTION_COUNT_PER_BATCH.skeleton); }); test.serial("Save actions should be chunked after compacting (3/3)", (t) => { @@ -108,12 +105,12 @@ test.serial("Save actions should be chunked after compacting (3/3)", (t) => { const trees = generateDummyTrees(1, nodeCount); Store.dispatch(addTreesAndGroupsAction(createTreeMapFromTreeArray(trees), [])); Store.dispatch(discardSaveQueuesAction()); - t.deepEqual(Store.getState().save.queue.skeleton, []); + t.deepEqual(Store.getState().save.queue, []); // Delete some node, NOTE that this is not the node in the middle of the tree! // The addTreesAndGroupsAction gives new ids to nodes and edges in a non-deterministic way. const middleNodeId = trees[0].nodes[nodeCount / 2].id; Store.dispatch(deleteNodeAction(middleNodeId)); - const { skeleton: skeletonSaveQueue } = Store.getState().save.queue; + const skeletonSaveQueue = Store.getState().save.queue; // There should only be one chunk t.is(skeletonSaveQueue.length, 1); t.true(skeletonSaveQueue[0].actions.length < MAXIMUM_ACTION_COUNT_PER_BATCH.skeleton); From b15405ef511d334dd1d76c1f80f565aef042ca96 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Mon, 16 Sep 2024 14:06:34 +0200 Subject: [PATCH 050/150] also store only one lastSaved and isBusy info for whole annotation --- frontend/javascripts/oxalis/default_state.ts | 12 +-- frontend/javascripts/oxalis/model.ts | 6 +- .../oxalis/model/accessors/save_accessor.ts | 11 --- .../oxalis/model/actions/save_actions.ts | 12 +-- .../oxalis/model/reducers/save_reducer.ts | 33 +------ .../oxalis/model/sagas/save_saga.ts | 95 +++++++++---------- frontend/javascripts/oxalis/store.ts | 13 +-- .../oxalis/view/action-bar/save_button.tsx | 13 ++- .../test/reducers/save_reducer.spec.ts | 12 +-- .../javascripts/test/sagas/save_saga.spec.ts | 26 ++--- 10 files changed, 70 insertions(+), 163 deletions(-) delete mode 100644 frontend/javascripts/oxalis/model/accessors/save_accessor.ts diff --git a/frontend/javascripts/oxalis/default_state.ts b/frontend/javascripts/oxalis/default_state.ts index 090a3384aa6..6124b8cdfa7 100644 --- a/frontend/javascripts/oxalis/default_state.ts +++ b/frontend/javascripts/oxalis/default_state.ts @@ -180,16 +180,8 @@ const defaultState: OxalisState = { }, save: { queue: [], - isBusyInfo: { - skeleton: false, - volumes: {}, - mappings: {}, - }, - lastSaveTimestamp: { - skeleton: 0, - volumes: {}, - mappings: {}, - }, + isBusy: false, + lastSaveTimestamp: 0, progressInfo: { processedActionCount: 0, totalActionCount: 0, diff --git a/frontend/javascripts/oxalis/model.ts b/frontend/javascripts/oxalis/model.ts index f2d9d5db78b..f3c7e63e7bf 100644 --- a/frontend/javascripts/oxalis/model.ts +++ b/frontend/javascripts/oxalis/model.ts @@ -9,7 +9,6 @@ import { isLayerVisible, } from "oxalis/model/accessors/dataset_accessor"; import { getTotalSaveQueueLength } from "oxalis/model/reducers/save_reducer"; -import { isBusy } from "oxalis/model/accessors/save_accessor"; import { isDatasetAccessibleBySwitching } from "admin/admin_rest_api"; import { saveNowAction } from "oxalis/model/actions/save_actions"; import type DataCube from "oxalis/model/bucket_data_handling/data_cube"; @@ -283,8 +282,7 @@ export class OxalisModel { stateSaved() { const state = Store.getState(); - const storeStateSaved = - !isBusy(state.save.isBusyInfo) && getTotalSaveQueueLength(state.save.queue) === 0; + const storeStateSaved = !state.save.isBusy && getTotalSaveQueueLength(state.save.queue) === 0; const pushQueuesSaved = _.reduce( this.dataLayers, @@ -341,7 +339,7 @@ export class OxalisModel { // The dispatch of the saveNowAction IN the while loop is deliberate. // Otherwise if an update action is pushed to the save queue during the Utils.sleep, // the while loop would continue running until the next save would be triggered. - if (!isBusy(Store.getState().save.isBusyInfo)) { + if (!Store.getState().save.isBusy) { Store.dispatch(saveNowAction()); } diff --git a/frontend/javascripts/oxalis/model/accessors/save_accessor.ts b/frontend/javascripts/oxalis/model/accessors/save_accessor.ts deleted file mode 100644 index d9dd9d65aa0..00000000000 --- a/frontend/javascripts/oxalis/model/accessors/save_accessor.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { IsBusyInfo, OxalisState, SaveQueueEntry } from "oxalis/store"; -import type { SaveQueueType } from "oxalis/model/actions/save_actions"; -import * as Utils from "libs/utils"; - -export function isBusy(isBusyInfo: IsBusyInfo): boolean { - return ( - isBusyInfo.skeleton || - Utils.values(isBusyInfo.volumes).some((el) => el) || - Utils.values(isBusyInfo.mappings).some((el) => el) - ); -} diff --git a/frontend/javascripts/oxalis/model/actions/save_actions.ts b/frontend/javascripts/oxalis/model/actions/save_actions.ts index dca4997b9f6..05f0a1680f9 100644 --- a/frontend/javascripts/oxalis/model/actions/save_actions.ts +++ b/frontend/javascripts/oxalis/model/actions/save_actions.ts @@ -64,24 +64,16 @@ export const discardSaveQueuesAction = () => type: "DISCARD_SAVE_QUEUES", }) as const; -export const setSaveBusyAction = ( - isBusy: boolean, - saveQueueType: SaveQueueType, - tracingId: string, -) => +export const setSaveBusyAction = (isBusy: boolean) => ({ type: "SET_SAVE_BUSY", isBusy, - saveQueueType, - tracingId, }) as const; -export const setLastSaveTimestampAction = (saveQueueType: SaveQueueType, tracingId: string) => +export const setLastSaveTimestampAction = () => ({ type: "SET_LAST_SAVE_TIMESTAMP", timestamp: Date.now(), - saveQueueType, - tracingId, }) as const; export const setVersionNumberAction = ( diff --git a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts index b8d75ff86dc..9b022ce96c5 100644 --- a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts +++ b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts @@ -81,32 +81,6 @@ function updateVersion(state: OxalisState, action: SetVersionNumberAction) { return state; } -function updateLastSaveTimestamp(state: OxalisState, action: SetLastSaveTimestampAction) { - if (action.saveQueueType === "skeleton") { - return updateKey2(state, "save", "lastSaveTimestamp", { - skeleton: action.timestamp, - }); - } else if (action.saveQueueType === "volume") { - const newVolumesDict = { - ...state.save.lastSaveTimestamp.volumes, - [action.tracingId]: action.timestamp, - }; - return updateKey2(state, "save", "lastSaveTimestamp", { - volumes: newVolumesDict, - }); - } else if (action.saveQueueType === "mapping") { - const newMappingsDict = { - ...state.save.lastSaveTimestamp.mappings, - [action.tracingId]: action.timestamp, - }; - return updateKey2(state, "save", "lastSaveTimestamp", { - mappings: newMappingsDict, - }); - } - - return state; -} - function SaveReducer(state: OxalisState, action: Action): OxalisState { switch (action.type) { case "PUSH_SAVE_QUEUE_TRANSACTION": { @@ -243,18 +217,17 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState { } case "SET_SAVE_BUSY": { - const newIsBusyInfo = updateTracingDict(action, state.save.isBusyInfo, action.isBusy); return update(state, { save: { - isBusyInfo: { - $set: newIsBusyInfo, + isBusy: { + $set: action.isBusy, }, }, }); } case "SET_LAST_SAVE_TIMESTAMP": { - return updateLastSaveTimestamp(state, action); + return updateKey2(state, "save", "lastSaveTimestamp", action.timestamp); } case "SET_VERSION_NUMBER": { diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts index 4cc5d6e6f5b..b6aa7eab05f 100644 --- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts @@ -57,10 +57,10 @@ import { call, delay, fork, put, race, take, takeEvery } from "typed-redux-saga" const ONE_YEAR_MS = 365 * 24 * 3600 * 1000; -export function* pushSaveQueueAsync(saveQueueType: SaveQueueType, tracingId: string): Saga { +export function* pushSaveQueueAsync(): Saga { yield* call(ensureWkReady); - yield* put(setLastSaveTimestampAction(saveQueueType, tracingId)); + yield* put(setLastSaveTimestampAction()); let loopCounter = 0; while (true) { @@ -85,7 +85,7 @@ export function* pushSaveQueueAsync(saveQueueType: SaveQueueType, tracingId: str timeout: delay(PUSH_THROTTLE_TIME), forcePush: take("SAVE_NOW"), }); - yield* put(setSaveBusyAction(true, saveQueueType, tracingId)); + yield* put(setSaveBusyAction(true)); // Send (parts) of the save queue to the server. // There are two main cases: @@ -119,7 +119,7 @@ export function* pushSaveQueueAsync(saveQueueType: SaveQueueType, tracingId: str } } - yield* put(setSaveBusyAction(false, saveQueueType, tracingId)); + yield* put(setSaveBusyAction(false)); } } export function sendRequestWithToken( @@ -209,18 +209,16 @@ export function* sendRequestToServer( } yield* put(setVersionNumberAction(version + versionIncrement, saveQueueType, tracingId)); - yield* put(setLastSaveTimestampAction(saveQueueType, tracingId)); + yield* put(setLastSaveTimestampAction()); yield* put(shiftSaveQueueAction(saveQueue.length, saveQueueType, tracingId)); - if (saveQueueType === "volume") { - try { - yield* call(markBucketsAsNotDirty, compactedSaveQueue, tracingId); - } catch (error) { - // If markBucketsAsNotDirty fails some reason, wk cannot recover from this error. - console.warn("Error when marking buckets as clean. No retry possible. Error:", error); - exceptionDuringMarkBucketsAsNotDirty = true; - throw error; - } + try { + yield* call(markBucketsAsNotDirty, compactedSaveQueue); + } catch (error) { + // If markBucketsAsNotDirty fails some reason, wk cannot recover from this error. + console.warn("Error when marking buckets as clean. No retry possible. Error:", error); + exceptionDuringMarkBucketsAsNotDirty = true; + throw error; } yield* call(toggleErrorHighlighting, false); @@ -285,33 +283,37 @@ export function* sendRequestToServer( } } -function* markBucketsAsNotDirty(saveQueue: Array, tracingId: string) { - const segmentationLayer = Model.getSegmentationTracingLayer(tracingId); - const segmentationResolutionInfo = yield* call(getResolutionInfo, segmentationLayer.resolutions); - - if (segmentationLayer != null) { - for (const saveEntry of saveQueue) { - for (const updateAction of saveEntry.actions) { - if (updateAction.name === "updateBucket") { - const { position, mag, additionalCoordinates } = updateAction.value; - const resolutionIndex = segmentationResolutionInfo.getIndexByResolution(mag); - const zoomedBucketAddress = globalPositionToBucketPosition( - position, - segmentationResolutionInfo.getDenseResolutions(), - resolutionIndex, - additionalCoordinates, - ); - const bucket = segmentationLayer.cube.getOrCreateBucket(zoomedBucketAddress); - - if (bucket.type === "null") { - continue; - } - - bucket.dirtyCount--; - - if (bucket.dirtyCount === 0) { - bucket.markAsPushed(); - } +function* markBucketsAsNotDirty(saveQueue: Array) { + for (const saveEntry of saveQueue) { + for (const updateAction of saveEntry.actions) { + if (updateAction.name === "updateBucket") { + // The ID must belong to a segmentation layer because we are handling + // an updateBucket action. + const { actionTracingId: tracingId } = updateAction.value; + const segmentationLayer = Model.getSegmentationTracingLayer(tracingId); + const segmentationResolutionInfo = yield* call( + getResolutionInfo, + segmentationLayer.resolutions, + ); + + const { position, mag, additionalCoordinates } = updateAction.value; + const resolutionIndex = segmentationResolutionInfo.getIndexByResolution(mag); + const zoomedBucketAddress = globalPositionToBucketPosition( + position, + segmentationResolutionInfo.getDenseResolutions(), + resolutionIndex, + additionalCoordinates, + ); + const bucket = segmentationLayer.cube.getOrCreateBucket(zoomedBucketAddress); + + if (bucket.type === "null") { + continue; + } + + bucket.dirtyCount--; + + if (bucket.dirtyCount === 0) { + bucket.markAsPushed(); } } } @@ -379,19 +381,11 @@ export function performDiffTracing( } export function* saveTracingAsync(): Saga { + yield* fork(pushSaveQueueAsync); yield* takeEvery("INITIALIZE_SKELETONTRACING", setupSavingForTracingType); yield* takeEvery("INITIALIZE_VOLUMETRACING", setupSavingForTracingType); - yield* takeEvery("INITIALIZE_EDITABLE_MAPPING", setupSavingForEditableMapping); } -export function* setupSavingForEditableMapping( - initializeAction: InitializeEditableMappingAction, -): Saga { - // No diffing needs to be done for editable mappings as the saga pushes update actions - // to the respective save queues, itself - const volumeTracingId = initializeAction.mapping.tracingId; - yield* fork(pushSaveQueueAsync, "mapping", volumeTracingId); -} export function* setupSavingForTracingType( initializeAction: InitializeSkeletonTracingAction | InitializeVolumeTracingAction, ): Saga { @@ -403,7 +397,6 @@ export function* setupSavingForTracingType( const saveQueueType = initializeAction.type === "INITIALIZE_SKELETONTRACING" ? "skeleton" : "volume"; const tracingId = initializeAction.tracing.id; - yield* fork(pushSaveQueueAsync, saveQueueType, tracingId); let prevTracing = (yield* select((state) => selectTracing(state, saveQueueType, tracingId))) as | VolumeTracing | SkeletonTracing; diff --git a/frontend/javascripts/oxalis/store.ts b/frontend/javascripts/oxalis/store.ts index 24aca44ec1f..720989e8abc 100644 --- a/frontend/javascripts/oxalis/store.ts +++ b/frontend/javascripts/oxalis/store.ts @@ -450,19 +450,10 @@ export type ProgressInfo = { readonly processedActionCount: number; readonly totalActionCount: number; }; -export type IsBusyInfo = { - readonly skeleton: boolean; - readonly volumes: Record; - readonly mappings: Record; -}; export type SaveState = { - readonly isBusyInfo: IsBusyInfo; + readonly isBusy: boolean; readonly queue: Array; - readonly lastSaveTimestamp: { - readonly skeleton: number; - readonly volumes: Record; - readonly mappings: Record; - }; + readonly lastSaveTimestamp: number; readonly progressInfo: ProgressInfo; }; export type Flycam = { diff --git a/frontend/javascripts/oxalis/view/action-bar/save_button.tsx b/frontend/javascripts/oxalis/view/action-bar/save_button.tsx index 87ad0d413fb..ab12ab6fd61 100644 --- a/frontend/javascripts/oxalis/view/action-bar/save_button.tsx +++ b/frontend/javascripts/oxalis/view/action-bar/save_button.tsx @@ -2,8 +2,7 @@ import { connect } from "react-redux"; import React from "react"; import _ from "lodash"; import Store, { type SaveState } from "oxalis/store"; -import type { OxalisState, IsBusyInfo } from "oxalis/store"; -import { isBusy } from "oxalis/model/accessors/save_accessor"; +import type { OxalisState } from "oxalis/store"; import ButtonComponent from "oxalis/view/components/button_component"; import { Model } from "oxalis/singletons"; import window from "libs/window"; @@ -25,7 +24,7 @@ type OwnProps = { }; type StateProps = { progressFraction: number | null | undefined; - isBusyInfo: IsBusyInfo; + isBusy: boolean; }; type Props = OwnProps & StateProps; type State = { @@ -101,7 +100,7 @@ class SaveButton extends React.PureComponent { getSaveButtonIcon() { if (this.state.isStateSaved) { return ; - } else if (isBusy(this.props.isBusyInfo)) { + } else if (this.props.isBusy) { return ; } else { return ; @@ -109,7 +108,7 @@ class SaveButton extends React.PureComponent { } shouldShowProgress(): boolean { - return isBusy(this.props.isBusyInfo) && this.props.progressFraction != null; + return this.props.isBusy && this.props.progressFraction != null; } render() { @@ -190,9 +189,9 @@ function getOldestUnsavedTimestamp(saveQueue: SaveState["queue"]): number | null } function mapStateToProps(state: OxalisState): StateProps { - const { progressInfo, isBusyInfo } = state.save; + const { progressInfo, isBusy } = state.save; return { - isBusyInfo, + isBusy, // For a low action count, the progress info would show only for a very short amount of time. // Therefore, the progressFraction is set to null, if the count is low. progressFraction: diff --git a/frontend/javascripts/test/reducers/save_reducer.spec.ts b/frontend/javascripts/test/reducers/save_reducer.spec.ts index b7a83326ca1..ff5882e2cf6 100644 --- a/frontend/javascripts/test/reducers/save_reducer.spec.ts +++ b/frontend/javascripts/test/reducers/save_reducer.spec.ts @@ -24,17 +24,9 @@ const tracingId = "1234567890"; const initialState: { save: SaveState; activeUser: APIUser; tracing: EmptyObject } = { activeUser: dummyUser, save: { - isBusyInfo: { - skeleton: false, - volumes: {}, - mappings: {}, - }, + isBusy: false, queue: [], - lastSaveTimestamp: { - skeleton: 0, - volumes: {}, - mappings: {}, - }, + lastSaveTimestamp: 0, progressInfo: { processedActionCount: 0, totalActionCount: 0, diff --git a/frontend/javascripts/test/sagas/save_saga.spec.ts b/frontend/javascripts/test/sagas/save_saga.spec.ts index ed9f3146fa8..3c8dd47fe3b 100644 --- a/frontend/javascripts/test/sagas/save_saga.spec.ts +++ b/frontend/javascripts/test/sagas/save_saga.spec.ts @@ -96,7 +96,7 @@ test("SaveSaga should send update actions", (t) => { saga.next({ forcePush: SaveActions.saveNowAction(), }), - put(setSaveBusyAction(true, TRACING_TYPE, tracingId)), + put(setSaveBusyAction(true)), ); saga.next(); // advance to next select state @@ -104,7 +104,7 @@ test("SaveSaga should send update actions", (t) => { expectValueDeepEqual(t, saga.next(saveQueue), call(sendRequestToServer, TRACING_TYPE, tracingId)); saga.next(saveQueue.length); // select state - expectValueDeepEqual(t, saga.next([]), put(setSaveBusyAction(false, TRACING_TYPE, tracingId))); + expectValueDeepEqual(t, saga.next([]), put(setSaveBusyAction(false))); // Test that loop repeats saga.next(); // select state @@ -229,7 +229,7 @@ test("SaveSaga should send update actions right away and try to reach a state wh saga.next(1); // advance to select state - expectValueDeepEqual(t, saga.next([]), put(setSaveBusyAction(false, TRACING_TYPE, tracingId))); + expectValueDeepEqual(t, saga.next([]), put(setSaveBusyAction(false))); }); test("SaveSaga should not try to reach state with all actions being saved when saving is triggered by a timeout", (t) => { const updateActions = [[UpdateActions.createEdge(1, 0, 1)], [UpdateActions.createEdge(1, 1, 2)]]; @@ -248,7 +248,7 @@ test("SaveSaga should not try to reach state with all actions being saved when s saga.next(saveQueue); // call sendRequestToServer - expectValueDeepEqual(t, saga.next([]), put(setSaveBusyAction(false, TRACING_TYPE, tracingId))); + expectValueDeepEqual(t, saga.next([]), put(setSaveBusyAction(false))); }); test("SaveSaga should remove the correct update actions", (t) => { const saveQueue = createSaveQueueFromUpdateActions( @@ -272,11 +272,7 @@ test("SaveSaga should remove the correct update actions", (t) => { saga.next(), put(SaveActions.setVersionNumberAction(3, TRACING_TYPE, tracingId)), ); - expectValueDeepEqual( - t, - saga.next(), - put(SaveActions.setLastSaveTimestampAction(TRACING_TYPE, tracingId)), - ); + expectValueDeepEqual(t, saga.next(), put(SaveActions.setLastSaveTimestampAction())); expectValueDeepEqual( t, saga.next(), @@ -306,11 +302,7 @@ test("SaveSaga should set the correct version numbers", (t) => { saga.next(), put(SaveActions.setVersionNumberAction(LAST_VERSION + 3, TRACING_TYPE, tracingId)), ); - expectValueDeepEqual( - t, - saga.next(), - put(SaveActions.setLastSaveTimestampAction(TRACING_TYPE, tracingId)), - ); + expectValueDeepEqual(t, saga.next(), put(SaveActions.setLastSaveTimestampAction())); expectValueDeepEqual( t, saga.next(), @@ -341,11 +333,7 @@ test("SaveSaga should set the correct version numbers if the save queue was comp saga.next(), put(SaveActions.setVersionNumberAction(LAST_VERSION + 1, TRACING_TYPE, tracingId)), ); - expectValueDeepEqual( - t, - saga.next(), - put(SaveActions.setLastSaveTimestampAction(TRACING_TYPE, tracingId)), - ); + expectValueDeepEqual(t, saga.next(), put(SaveActions.setLastSaveTimestampAction())); expectValueDeepEqual( t, saga.next(), From 3952474e3f1f1d272101095a87a0a930833a3194 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Mon, 16 Sep 2024 14:17:55 +0200 Subject: [PATCH 051/150] make sendRequestToServer independent of tracing --- .../oxalis/model/actions/save_actions.ts | 8 +--- .../oxalis/model/reducers/save_reducer.ts | 37 +------------------ .../oxalis/model/sagas/save_saga.ts | 29 ++++++++------- .../oxalis/model/sagas/save_saga_constants.ts | 12 +++--- 4 files changed, 25 insertions(+), 61 deletions(-) diff --git a/frontend/javascripts/oxalis/model/actions/save_actions.ts b/frontend/javascripts/oxalis/model/actions/save_actions.ts index 05f0a1680f9..42d8e47747c 100644 --- a/frontend/javascripts/oxalis/model/actions/save_actions.ts +++ b/frontend/javascripts/oxalis/model/actions/save_actions.ts @@ -47,16 +47,10 @@ export const saveNowAction = () => type: "SAVE_NOW", }) as const; -export const shiftSaveQueueAction = ( - count: number, - saveQueueType: SaveQueueType, - tracingId: string, -) => +export const shiftSaveQueueAction = (count: number) => ({ type: "SHIFT_SAVE_QUEUE", count, - saveQueueType, - tracingId, }) as const; export const discardSaveQueuesAction = () => diff --git a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts index 9b022ce96c5..26a9f0e19ce 100644 --- a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts +++ b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts @@ -1,12 +1,8 @@ import _ from "lodash"; import update from "immutability-helper"; import type { Action } from "oxalis/model/actions/actions"; -import type { OxalisState, SaveState, SaveQueueEntry } from "oxalis/store"; -import type { - SetVersionNumberAction, - SetLastSaveTimestampAction, - SaveQueueType, -} from "oxalis/model/actions/save_actions"; +import type { OxalisState, SaveState } from "oxalis/store"; +import type { SetVersionNumberAction } from "oxalis/model/actions/save_actions"; import { getActionLog } from "oxalis/model/helpers/action_logger_middleware"; import { getStats } from "oxalis/model/accessors/annotation_accessor"; import { MAXIMUM_ACTION_COUNT_PER_BATCH } from "oxalis/model/sagas/save_saga_constants"; @@ -16,7 +12,6 @@ import { updateVolumeTracing, } from "oxalis/model/reducers/volumetracing_reducer_helpers"; import Date from "libs/date"; -import * as Utils from "libs/utils"; import type { UpdateAction, UpdateActionWithTracingId } from "../sagas/update_actions"; // These update actions are not idempotent. Having them @@ -31,34 +26,6 @@ const NOT_IDEMPOTENT_ACTIONS = [ "deleteNode", ]; -type TracingDict = { - skeleton: V; - volumes: Record; - mappings: Record; -}; - -function updateTracingDict( - action: { saveQueueType: SaveQueueType; tracingId: string }, - oldDict: TracingDict, - newValue: V, -): TracingDict { - if (action.saveQueueType === "skeleton") { - return { ...oldDict, skeleton: newValue }; - } else if (action.saveQueueType === "volume") { - return { - ...oldDict, - volumes: { ...oldDict.volumes, [action.tracingId]: newValue }, - }; - } else if (action.saveQueueType === "mapping") { - return { - ...oldDict, - mappings: { ...oldDict.mappings, [action.tracingId]: newValue }, - }; - } - - return oldDict; -} - export function getTotalSaveQueueLength(queueObj: SaveState["queue"]) { return queueObj.length; } diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts index b6aa7eab05f..0916eca859c 100644 --- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts @@ -113,7 +113,7 @@ export function* pushSaveQueueAsync(): Saga { saveQueue = yield* select((state) => state.save.queue); if (saveQueue.length > 0) { - savedItemCount += yield* call(sendRequestToServer, saveQueueType, tracingId); + savedItemCount += yield* call(sendRequestToServer); } else { break; } @@ -131,17 +131,14 @@ export function sendRequestWithToken( // This function returns the first n batches of the provided array, so that the count of // all actions in these n batches does not exceed MAXIMUM_ACTION_COUNT_PER_SAVE -function sliceAppropriateBatchCount( - batches: Array, - saveQueueType: SaveQueueType, -): Array { +function sliceAppropriateBatchCount(batches: Array): Array { const slicedBatches = []; let actionCount = 0; for (const batch of batches) { const newActionCount = actionCount + batch.actions.length; - if (newActionCount <= MAXIMUM_ACTION_COUNT_PER_SAVE[saveQueueType]) { + if (newActionCount <= MAXIMUM_ACTION_COUNT_PER_SAVE) { actionCount = newActionCount; slicedBatches.push(batch); } else { @@ -161,10 +158,7 @@ function getRetryWaitTime(retryCount: number) { // at any time, because the browser page is reloaded after the message is shown, anyway. let didShowFailedSimultaneousTracingError = false; -export function* sendRequestToServer( - saveQueueType: SaveQueueType, - tracingId: string, -): Saga { +export function* sendRequestToServer(): Saga { /* * Saves a reasonably-sized part of the save queue (that corresponds to the * tracingId) to the server (plus retry-mechanism). @@ -172,9 +166,12 @@ export function* sendRequestToServer( */ const fullSaveQueue = yield* select((state) => state.save.queue); - const saveQueue = sliceAppropriateBatchCount(fullSaveQueue, saveQueueType); + const saveQueue = sliceAppropriateBatchCount(fullSaveQueue); let compactedSaveQueue = compactSaveQueue(saveQueue); - const { version } = yield* select((state) => selectTracing(state, saveQueueType, tracingId)); + const tracings = yield* select((state) => + _.compact([state.tracing.skeleton, ...state.tracing.volumes, ...state.tracing.mappings]), + ); + const version = _.max(tracings.map((t) => t.version)) || 0; const annotationId = yield* select((state) => state.tracing.annotationId); const tracingStoreUrl = yield* select((state) => state.tracing.tracingStore.url); let versionIncrement; @@ -208,9 +205,13 @@ export function* sendRequestToServer( ); } - yield* put(setVersionNumberAction(version + versionIncrement, saveQueueType, tracingId)); + for (const tracing of tracings) { + yield* put( + setVersionNumberAction(version + versionIncrement, tracing.type, tracing.tracingId), + ); + } yield* put(setLastSaveTimestampAction()); - yield* put(shiftSaveQueueAction(saveQueue.length, saveQueueType, tracingId)); + yield* put(shiftSaveQueueAction(saveQueue.length)); try { yield* call(markBucketsAsNotDirty, compactedSaveQueue); diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga_constants.ts b/frontend/javascripts/oxalis/model/sagas/save_saga_constants.ts index 0fdc776eb2c..87ace921de4 100644 --- a/frontend/javascripts/oxalis/model/sagas/save_saga_constants.ts +++ b/frontend/javascripts/oxalis/model/sagas/save_saga_constants.ts @@ -17,8 +17,10 @@ export const MAXIMUM_ACTION_COUNT_PER_BATCH = { mapping: Number.POSITIVE_INFINITY, // The back-end does not accept transactions for mappings. } as const; -export const MAXIMUM_ACTION_COUNT_PER_SAVE = { - skeleton: 15000, - volume: 3000, - mapping: Number.POSITIVE_INFINITY, // The back-end does not accept transactions for mappings. -} as const; +// todop: should this be smarter? +// export const MAXIMUM_ACTION_COUNT_PER_SAVE = { +// skeleton: 15000, +// volume: 3000, +// mapping: Number.POSITIVE_INFINITY, // The back-end does not accept transactions for mappings. +// } as const; +export const MAXIMUM_ACTION_COUNT_PER_SAVE = 3000; From 48bf1b7f6f2a4694c69ca271a703b5db8a2beb2d Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Mon, 16 Sep 2024 14:18:54 +0200 Subject: [PATCH 052/150] rename sendRequestToServer --- .../oxalis/model/sagas/save_saga.ts | 7 +++--- .../javascripts/test/sagas/save_saga.spec.ts | 24 +++++++++++-------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts index 0916eca859c..44d3eae1b5e 100644 --- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts @@ -42,7 +42,6 @@ import { } from "oxalis/model/sagas/save_saga_constants"; import { diffSkeletonTracing } from "oxalis/model/sagas/skeletontracing_saga"; import type { UpdateAction } from "oxalis/model/sagas/update_actions"; -import { updateTdCamera } from "oxalis/model/sagas/update_actions"; import { diffVolumeTracing } from "oxalis/model/sagas/volumetracing_saga"; import { ensureWkReady } from "oxalis/model/sagas/wk_ready_saga"; import { Model } from "oxalis/singletons"; @@ -67,7 +66,7 @@ export function* pushSaveQueueAsync(): Saga { loopCounter++; let saveQueue; // Check whether the save queue is actually empty, the PUSH_SAVE_QUEUE_TRANSACTION action - // could have been triggered during the call to sendRequestToServer + // could have been triggered during the call to sendSaveRequestToServer saveQueue = yield* select((state) => state.save.queue); if (saveQueue.length === 0) { @@ -113,7 +112,7 @@ export function* pushSaveQueueAsync(): Saga { saveQueue = yield* select((state) => state.save.queue); if (saveQueue.length > 0) { - savedItemCount += yield* call(sendRequestToServer); + savedItemCount += yield* call(sendSaveRequestToServer); } else { break; } @@ -158,7 +157,7 @@ function getRetryWaitTime(retryCount: number) { // at any time, because the browser page is reloaded after the message is shown, anyway. let didShowFailedSimultaneousTracingError = false; -export function* sendRequestToServer(): Saga { +export function* sendSaveRequestToServer(): Saga { /* * Saves a reasonably-sized part of the save queue (that corresponds to the * tracingId) to the server (plus retry-mechanism). diff --git a/frontend/javascripts/test/sagas/save_saga.spec.ts b/frontend/javascripts/test/sagas/save_saga.spec.ts index 3c8dd47fe3b..c85580c719d 100644 --- a/frontend/javascripts/test/sagas/save_saga.spec.ts +++ b/frontend/javascripts/test/sagas/save_saga.spec.ts @@ -23,7 +23,7 @@ const SaveActions = mockRequire.reRequire("oxalis/model/actions/save_actions"); const { take, call, put } = mockRequire.reRequire("redux-saga/effects"); const { pushSaveQueueAsync, - sendRequestToServer, + sendSaveRequestToServer, toggleErrorHighlighting, addVersionNumbers, sendRequestWithToken, @@ -101,7 +101,11 @@ test("SaveSaga should send update actions", (t) => { saga.next(); // advance to next select state - expectValueDeepEqual(t, saga.next(saveQueue), call(sendRequestToServer, TRACING_TYPE, tracingId)); + expectValueDeepEqual( + t, + saga.next(saveQueue), + call(sendSaveRequestToServer, TRACING_TYPE, tracingId), + ); saga.next(saveQueue.length); // select state expectValueDeepEqual(t, saga.next([]), put(setSaveBusyAction(false))); @@ -116,7 +120,7 @@ test("SaveSaga should send request to server", (t) => { TIMESTAMP, tracingId, ); - const saga = sendRequestToServer(TRACING_TYPE, tracingId); + const saga = sendSaveRequestToServer(TRACING_TYPE, tracingId); saga.next(); saga.next(saveQueue); saga.next({ @@ -152,7 +156,7 @@ test("SaveSaga should retry update actions", (t) => { compress: false, }, ); - const saga = sendRequestToServer(TRACING_TYPE, tracingId); + const saga = sendSaveRequestToServer(TRACING_TYPE, tracingId); saga.next(); saga.next(saveQueue); saga.next({ @@ -175,7 +179,7 @@ test("SaveSaga should escalate on permanent client error update actions", (t) => TIMESTAMP, tracingId, ); - const saga = sendRequestToServer(TRACING_TYPE, tracingId); + const saga = sendSaveRequestToServer(TRACING_TYPE, tracingId); saga.next(); saga.next(saveQueue); saga.next({ @@ -225,7 +229,7 @@ test("SaveSaga should send update actions right away and try to reach a state wh saga.next(); // select state - saga.next(saveQueue); // call sendRequestToServer + saga.next(saveQueue); // call sendSaveRequestToServer saga.next(1); // advance to select state @@ -246,7 +250,7 @@ test("SaveSaga should not try to reach state with all actions being saved when s timeout: "a placeholder", }); // put setSaveBusyAction - saga.next(saveQueue); // call sendRequestToServer + saga.next(saveQueue); // call sendSaveRequestToServer expectValueDeepEqual(t, saga.next([]), put(setSaveBusyAction(false))); }); @@ -259,7 +263,7 @@ test("SaveSaga should remove the correct update actions", (t) => { TIMESTAMP, tracingId, ); - const saga = sendRequestToServer(TRACING_TYPE, tracingId); + const saga = sendSaveRequestToServer(TRACING_TYPE, tracingId); saga.next(); saga.next(saveQueue); saga.next({ @@ -289,7 +293,7 @@ test("SaveSaga should set the correct version numbers", (t) => { TIMESTAMP, tracingId, ); - const saga = sendRequestToServer(TRACING_TYPE, tracingId); + const saga = sendSaveRequestToServer(TRACING_TYPE, tracingId); saga.next(); saga.next(saveQueue); saga.next({ @@ -319,7 +323,7 @@ test("SaveSaga should set the correct version numbers if the save queue was comp TIMESTAMP, tracingId, ); - const saga = sendRequestToServer(TRACING_TYPE, tracingId); + const saga = sendSaveRequestToServer(TRACING_TYPE, tracingId); saga.next(); saga.next(saveQueue); saga.next({ From cd101bda3f8a90f7fc7bd1d0fbbe316162ea76f8 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Mon, 16 Sep 2024 14:51:01 +0200 Subject: [PATCH 053/150] fix save_reducer.spec --- .../oxalis/model/sagas/mapping_saga.ts | 4 +- .../oxalis/model/sagas/save_saga.ts | 2 - .../test/reducers/save_reducer.spec.ts | 63 ++++++++++--------- 3 files changed, 37 insertions(+), 32 deletions(-) diff --git a/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts b/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts index 0ec9d30ab66..1a0044c4370 100644 --- a/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts @@ -96,7 +96,9 @@ const takeLatestMappingChange = ( ); const mapping = getMappingInfo(activeMappingByLayer, layerName); - console.log("Changed from", lastBucketRetrievalSource, "to", bucketRetrievalSource); + if (process.env.NODE_ENV === "production") { + console.log("Changed from", lastBucketRetrievalSource, "to", bucketRetrievalSource); + } if (lastWatcherTask) { console.log("Cancel old bucket watcher"); diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts index 44d3eae1b5e..d9253fc6606 100644 --- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts @@ -13,7 +13,6 @@ import { getResolutionInfo } from "oxalis/model/accessors/dataset_accessor"; import { selectTracing } from "oxalis/model/accessors/tracing_accessor"; import { getVolumeTracingById } from "oxalis/model/accessors/volumetracing_accessor"; import { FlycamActions } from "oxalis/model/actions/flycam_actions"; -import type { SaveQueueType } from "oxalis/model/actions/save_actions"; import { pushSaveQueueTransaction, setLastSaveTimestampAction, @@ -25,7 +24,6 @@ import type { InitializeSkeletonTracingAction } from "oxalis/model/actions/skele import { SkeletonTracingSaveRelevantActions } from "oxalis/model/actions/skeletontracing_actions"; import { ViewModeSaveRelevantActions } from "oxalis/model/actions/view_mode_actions"; import { - type InitializeEditableMappingAction, type InitializeVolumeTracingAction, VolumeTracingSaveRelevantActions, } from "oxalis/model/actions/volumetracing_actions"; diff --git a/frontend/javascripts/test/reducers/save_reducer.spec.ts b/frontend/javascripts/test/reducers/save_reducer.spec.ts index ff5882e2cf6..6428fb94bd8 100644 --- a/frontend/javascripts/test/reducers/save_reducer.spec.ts +++ b/frontend/javascripts/test/reducers/save_reducer.spec.ts @@ -2,10 +2,9 @@ import mockRequire from "mock-require"; import test from "ava"; import "test/reducers/save_reducer.mock"; import dummyUser from "test/fixtures/dummy_user"; -import type { SaveState } from "oxalis/store"; -import type { APIUser } from "types/api_flow_types"; +import type { OxalisState } from "oxalis/store"; import { createSaveQueueFromUpdateActions } from "../helpers/saveHelpers"; -import type { EmptyObject } from "types/globals"; +import type { UpdateAction } from "oxalis/model/sagas/update_actions"; const TIMESTAMP = 1494695001688; const DateMock = { @@ -16,12 +15,18 @@ const AccessorMock = { }; mockRequire("libs/date", DateMock); mockRequire("oxalis/model/accessors/skeletontracing_accessor", AccessorMock); -const SaveActions = mockRequire.reRequire("oxalis/model/actions/save_actions"); -const SaveReducer = mockRequire.reRequire("oxalis/model/reducers/save_reducer").default; -const { createEdge } = mockRequire.reRequire("oxalis/model/sagas/update_actions"); + +const SaveActions = mockRequire.reRequire( + "oxalis/model/actions/save_actions", +) as typeof import("oxalis/model/actions/save_actions"); +const SaveReducer = mockRequire.reRequire("oxalis/model/reducers/save_reducer") + .default as typeof import("oxalis/model/reducers/save_reducer")["default"]; +const { createEdge } = mockRequire.reRequire( + "oxalis/model/sagas/update_actions", +) as typeof import("oxalis/model/sagas/update_actions"); const tracingId = "1234567890"; -const initialState: { save: SaveState; activeUser: APIUser; tracing: EmptyObject } = { +const initialState = { activeUser: dummyUser, save: { isBusy: false, @@ -33,13 +38,13 @@ const initialState: { save: SaveState; activeUser: APIUser; tracing: EmptyObject }, }, tracing: {}, -}; +} as any as OxalisState; test("Save should add update actions to the queue", (t) => { const items = [createEdge(0, 1, 2), createEdge(0, 2, 3)]; const saveQueue = createSaveQueueFromUpdateActions([items], TIMESTAMP, tracingId); - const pushAction = SaveActions.pushSaveQueueTransaction(items, "skeleton"); + const pushAction = SaveActions.pushSaveQueueTransaction(items, "skeleton", tracingId); const newState = SaveReducer(initialState, pushAction); - t.deepEqual(newState.save.queue.skeleton, saveQueue); + t.deepEqual(newState.save.queue, saveQueue); }); test("Save should add more update actions to the queue", (t) => { const getItems = (treeId: number) => [createEdge(treeId, 1, 2), createEdge(treeId, 2, 3)]; @@ -50,54 +55,54 @@ test("Save should add more update actions to the queue", (t) => { ); const testState = SaveReducer( initialState, - SaveActions.pushSaveQueueTransaction(getItems(0), "skeleton"), + SaveActions.pushSaveQueueTransaction(getItems(0), "skeleton", tracingId), ); const newState = SaveReducer( testState, - SaveActions.pushSaveQueueTransaction(getItems(1), "skeleton"), + SaveActions.pushSaveQueueTransaction(getItems(1), "skeleton", tracingId), ); - t.deepEqual(newState.save.queue.skeleton, saveQueue); + t.deepEqual(newState.save.queue, saveQueue); }); test("Save should add zero update actions to the queue", (t) => { - const items = [] as const; - const pushAction = SaveActions.pushSaveQueueTransaction(items, "skeleton"); + const items: UpdateAction[] = []; + const pushAction = SaveActions.pushSaveQueueTransaction(items, "skeleton", tracingId); const newState = SaveReducer(initialState, pushAction); - t.deepEqual(newState.save.queue.skeleton, []); + t.deepEqual(newState.save.queue, []); }); test("Save should remove one update actions from the queue", (t) => { const firstItem = [createEdge(0, 1, 2)]; const secondItem = [createEdge(1, 2, 3)]; const saveQueue = createSaveQueueFromUpdateActions([secondItem], TIMESTAMP, tracingId); - const firstPushAction = SaveActions.pushSaveQueueTransaction(firstItem, "skeleton"); - const secondPushAction = SaveActions.pushSaveQueueTransaction(secondItem, "skeleton"); - const popAction = SaveActions.shiftSaveQueueAction(1, "skeleton"); + const firstPushAction = SaveActions.pushSaveQueueTransaction(firstItem, "skeleton", tracingId); + const secondPushAction = SaveActions.pushSaveQueueTransaction(secondItem, "skeleton", tracingId); + const popAction = SaveActions.shiftSaveQueueAction(1); let newState = SaveReducer(initialState, firstPushAction); newState = SaveReducer(newState, secondPushAction); newState = SaveReducer(newState, popAction); - t.deepEqual(newState.save.queue.skeleton, saveQueue); + t.deepEqual(newState.save.queue, saveQueue); }); test("Save should remove zero update actions from the queue", (t) => { const items = [createEdge(0, 1, 2), createEdge(1, 2, 3)]; const saveQueue = createSaveQueueFromUpdateActions([items], TIMESTAMP, tracingId); - const pushAction = SaveActions.pushSaveQueueTransaction(items, "skeleton"); - const popAction = SaveActions.shiftSaveQueueAction(0, "skeleton"); + const pushAction = SaveActions.pushSaveQueueTransaction(items, "skeleton", tracingId); + const popAction = SaveActions.shiftSaveQueueAction(0); let newState = SaveReducer(initialState, pushAction); newState = SaveReducer(newState, popAction); - t.deepEqual(newState.save.queue.skeleton, saveQueue); + t.deepEqual(newState.save.queue, saveQueue); }); test("Save should remove all update actions from the queue (1/2)", (t) => { const items = [createEdge(0, 1, 2), createEdge(0, 2, 3)]; - const pushAction = SaveActions.pushSaveQueueTransaction(items, "skeleton"); - const popAction = SaveActions.shiftSaveQueueAction(2, "skeleton"); + const pushAction = SaveActions.pushSaveQueueTransaction(items, "skeleton", tracingId); + const popAction = SaveActions.shiftSaveQueueAction(2); let newState = SaveReducer(initialState, pushAction); newState = SaveReducer(newState, popAction); - t.deepEqual(newState.save.queue.skeleton, []); + t.deepEqual(newState.save.queue, []); }); test("Save should remove all update actions from the queue (2/2)", (t) => { const items = [createEdge(0, 1, 2), createEdge(0, 2, 3)]; - const pushAction = SaveActions.pushSaveQueueTransaction(items, "skeleton"); - const popAction = SaveActions.shiftSaveQueueAction(5, "skeleton"); + const pushAction = SaveActions.pushSaveQueueTransaction(items, "skeleton", tracingId); + const popAction = SaveActions.shiftSaveQueueAction(5); let newState = SaveReducer(initialState, pushAction); newState = SaveReducer(newState, popAction); - t.deepEqual(newState.save.queue.skeleton, []); + t.deepEqual(newState.save.queue, []); }); From b3fb1193df1bf3a4a42480de1558e909c97f2920 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Mon, 16 Sep 2024 15:25:18 +0200 Subject: [PATCH 054/150] fix save saga spec --- .../javascripts/test/sagas/save_saga.spec.ts | 178 ++++++++++-------- 1 file changed, 102 insertions(+), 76 deletions(-) diff --git a/frontend/javascripts/test/sagas/save_saga.spec.ts b/frontend/javascripts/test/sagas/save_saga.spec.ts index c85580c719d..8036007eb96 100644 --- a/frontend/javascripts/test/sagas/save_saga.spec.ts +++ b/frontend/javascripts/test/sagas/save_saga.spec.ts @@ -18,17 +18,27 @@ mockRequire("libs/date", DateMock); mockRequire("oxalis/model/sagas/root_saga", function* () { yield; }); -const UpdateActions = mockRequire.reRequire("oxalis/model/sagas/update_actions"); -const SaveActions = mockRequire.reRequire("oxalis/model/actions/save_actions"); -const { take, call, put } = mockRequire.reRequire("redux-saga/effects"); +const UpdateActions = mockRequire.reRequire( + "oxalis/model/sagas/update_actions", +) as typeof import("oxalis/model/sagas/update_actions"); +const SaveActions = mockRequire.reRequire( + "oxalis/model/actions/save_actions", +) as typeof import("oxalis/model/actions/save_actions"); +const { take, call, put } = mockRequire.reRequire( + "redux-saga/effects", +) as typeof import("redux-saga/effects"); const { pushSaveQueueAsync, sendSaveRequestToServer, toggleErrorHighlighting, addVersionNumbers, sendRequestWithToken, -} = mockRequire.reRequire("oxalis/model/sagas/save_saga"); -const tracingId = "1234567890"; +} = mockRequire.reRequire( + "oxalis/model/sagas/save_saga", +) as typeof import("oxalis/model/sagas/save_saga"); + +const annotationId = "annotation-abcdefgh"; +const tracingId = "tracing-1234567890"; const initialState = { dataset: { dataSource: { @@ -71,8 +81,8 @@ const TRACING_TYPE = "skeleton"; test("SaveSaga should compact multiple updateTracing update actions", (t) => { const saveQueue = createSaveQueueFromUpdateActions( [ - [UpdateActions.updateSkeletonTracing(initialState, [1, 2, 3], [0, 0, 1], 1)], - [UpdateActions.updateSkeletonTracing(initialState, [2, 3, 4], [0, 0, 1], 2)], + [UpdateActions.updateSkeletonTracing(initialState.tracing, [1, 2, 3], [], [0, 0, 1], 1)], + [UpdateActions.updateSkeletonTracing(initialState.tracing, [2, 3, 4], [], [0, 0, 1], 2)], ], TIMESTAMP, tracingId, @@ -82,7 +92,7 @@ test("SaveSaga should compact multiple updateTracing update actions", (t) => { test("SaveSaga should send update actions", (t) => { const updateActions = [[UpdateActions.createEdge(1, 0, 1)], [UpdateActions.createEdge(1, 1, 2)]]; const saveQueue = createSaveQueueFromUpdateActions(updateActions, TIMESTAMP, tracingId); - const saga = pushSaveQueueAsync(TRACING_TYPE, tracingId); + const saga = pushSaveQueueAsync(); expectValueDeepEqual(t, saga.next(), call(ensureWkReady)); saga.next(); // setLastSaveTimestampAction @@ -101,11 +111,7 @@ test("SaveSaga should send update actions", (t) => { saga.next(); // advance to next select state - expectValueDeepEqual( - t, - saga.next(saveQueue), - call(sendSaveRequestToServer, TRACING_TYPE, tracingId), - ); + expectValueDeepEqual(t, saga.next(saveQueue), call(sendSaveRequestToServer)); saga.next(saveQueue.length); // select state expectValueDeepEqual(t, saga.next([]), put(setSaveBusyAction(false))); @@ -120,23 +126,31 @@ test("SaveSaga should send request to server", (t) => { TIMESTAMP, tracingId, ); - const saga = sendSaveRequestToServer(TRACING_TYPE, tracingId); + const saga = sendSaveRequestToServer(); saga.next(); saga.next(saveQueue); - saga.next({ - version: LAST_VERSION, - type: TRACING_TYPE, - }); + saga.next([ + { + version: LAST_VERSION, + type: TRACING_TYPE, + tracingId, + }, + ]); + saga.next(annotationId); const [saveQueueWithVersions, versionIncrement] = addVersionNumbers(saveQueue, LAST_VERSION); t.is(versionIncrement, 2); expectValueDeepEqual( t, saga.next(TRACINGSTORE_URL), - call(sendRequestWithToken, `${TRACINGSTORE_URL}/tracings/skeleton/1234567890/update?token=`, { - method: "POST", - data: saveQueueWithVersions, - compress: false, - }), + call( + sendRequestWithToken, + `${TRACINGSTORE_URL}/tracings/annotation/${annotationId}/update?token=`, + { + method: "POST", + data: saveQueueWithVersions, + compress: false, + }, + ), ); }); test("SaveSaga should retry update actions", (t) => { @@ -149,20 +163,24 @@ test("SaveSaga should retry update actions", (t) => { t.is(versionIncrement, 2); const requestWithTokenCall = call( sendRequestWithToken, - `${TRACINGSTORE_URL}/tracings/skeleton/1234567890/update?token=`, + `${TRACINGSTORE_URL}/tracings/annotation/${annotationId}/update?token=`, { method: "POST", data: saveQueueWithVersions, compress: false, }, ); - const saga = sendSaveRequestToServer(TRACING_TYPE, tracingId); + const saga = sendSaveRequestToServer(); saga.next(); saga.next(saveQueue); - saga.next({ - version: LAST_VERSION, - type: TRACING_TYPE, - }); + saga.next([ + { + version: LAST_VERSION, + type: TRACING_TYPE, + tracingId, + }, + ]); + saga.next(annotationId); expectValueDeepEqual(t, saga.next(TRACINGSTORE_URL), requestWithTokenCall); saga.throw("Timeout"); expectValueDeepEqual(t, saga.next("Explorational"), call(toggleErrorHighlighting, true)); @@ -179,23 +197,31 @@ test("SaveSaga should escalate on permanent client error update actions", (t) => TIMESTAMP, tracingId, ); - const saga = sendSaveRequestToServer(TRACING_TYPE, tracingId); + const saga = sendSaveRequestToServer(); saga.next(); saga.next(saveQueue); - saga.next({ - version: LAST_VERSION, - type: TRACING_TYPE, - }); + saga.next([ + { + version: LAST_VERSION, + type: TRACING_TYPE, + tracingId, + }, + ]); + saga.next(annotationId); const [saveQueueWithVersions, versionIncrement] = addVersionNumbers(saveQueue, LAST_VERSION); t.is(versionIncrement, 2); expectValueDeepEqual( t, saga.next(TRACINGSTORE_URL), - call(sendRequestWithToken, `${TRACINGSTORE_URL}/tracings/skeleton/1234567890/update?token=`, { - method: "POST", - data: saveQueueWithVersions, - compress: false, - }), + call( + sendRequestWithToken, + `${TRACINGSTORE_URL}/tracings/annotation/${annotationId}/update?token=`, + { + method: "POST", + data: saveQueueWithVersions, + compress: false, + }, + ), ); saga.throw({ status: 409, @@ -215,7 +241,7 @@ test("SaveSaga should escalate on permanent client error update actions", (t) => test("SaveSaga should send update actions right away and try to reach a state where all updates are saved", (t) => { const updateActions = [[UpdateActions.createEdge(1, 0, 1)], [UpdateActions.createEdge(1, 1, 2)]]; const saveQueue = createSaveQueueFromUpdateActions(updateActions, TIMESTAMP, tracingId); - const saga = pushSaveQueueAsync(TRACING_TYPE, tracingId); + const saga = pushSaveQueueAsync(); expectValueDeepEqual(t, saga.next(), call(ensureWkReady)); saga.next(); saga.next(); // select state @@ -238,7 +264,7 @@ test("SaveSaga should send update actions right away and try to reach a state wh test("SaveSaga should not try to reach state with all actions being saved when saving is triggered by a timeout", (t) => { const updateActions = [[UpdateActions.createEdge(1, 0, 1)], [UpdateActions.createEdge(1, 1, 2)]]; const saveQueue = createSaveQueueFromUpdateActions(updateActions, TIMESTAMP, tracingId); - const saga = pushSaveQueueAsync(TRACING_TYPE, tracingId); + const saga = pushSaveQueueAsync(); expectValueDeepEqual(t, saga.next(), call(ensureWkReady)); saga.next(); saga.next(); // select state @@ -257,19 +283,23 @@ test("SaveSaga should not try to reach state with all actions being saved when s test("SaveSaga should remove the correct update actions", (t) => { const saveQueue = createSaveQueueFromUpdateActions( [ - [UpdateActions.updateSkeletonTracing(initialState, [1, 2, 3], [0, 0, 1], 1)], - [UpdateActions.updateSkeletonTracing(initialState, [2, 3, 4], [0, 0, 1], 2)], + [UpdateActions.updateSkeletonTracing(initialState.tracing, [1, 2, 3], [], [0, 0, 1], 1)], + [UpdateActions.updateSkeletonTracing(initialState.tracing, [2, 3, 4], [], [0, 0, 1], 2)], ], TIMESTAMP, tracingId, ); - const saga = sendSaveRequestToServer(TRACING_TYPE, tracingId); + const saga = sendSaveRequestToServer(); saga.next(); saga.next(saveQueue); - saga.next({ - version: LAST_VERSION, - type: TRACING_TYPE, - }); + saga.next([ + { + version: LAST_VERSION, + type: TRACING_TYPE, + tracingId, + }, + ]); + saga.next(annotationId); saga.next(TRACINGSTORE_URL); expectValueDeepEqual( t, @@ -277,11 +307,7 @@ test("SaveSaga should remove the correct update actions", (t) => { put(SaveActions.setVersionNumberAction(3, TRACING_TYPE, tracingId)), ); expectValueDeepEqual(t, saga.next(), put(SaveActions.setLastSaveTimestampAction())); - expectValueDeepEqual( - t, - saga.next(), - put(SaveActions.shiftSaveQueueAction(2, TRACING_TYPE, tracingId)), - ); + expectValueDeepEqual(t, saga.next(), put(SaveActions.shiftSaveQueueAction(2))); }); test("SaveSaga should set the correct version numbers", (t) => { const saveQueue = createSaveQueueFromUpdateActions( @@ -293,13 +319,17 @@ test("SaveSaga should set the correct version numbers", (t) => { TIMESTAMP, tracingId, ); - const saga = sendSaveRequestToServer(TRACING_TYPE, tracingId); + const saga = sendSaveRequestToServer(); saga.next(); saga.next(saveQueue); - saga.next({ - version: LAST_VERSION, - type: TRACING_TYPE, - }); + saga.next([ + { + version: LAST_VERSION, + type: TRACING_TYPE, + tracingId, + }, + ]); + saga.next(annotationId); saga.next(TRACINGSTORE_URL); expectValueDeepEqual( t, @@ -307,29 +337,29 @@ test("SaveSaga should set the correct version numbers", (t) => { put(SaveActions.setVersionNumberAction(LAST_VERSION + 3, TRACING_TYPE, tracingId)), ); expectValueDeepEqual(t, saga.next(), put(SaveActions.setLastSaveTimestampAction())); - expectValueDeepEqual( - t, - saga.next(), - put(SaveActions.shiftSaveQueueAction(3, TRACING_TYPE, tracingId)), - ); + expectValueDeepEqual(t, saga.next(), put(SaveActions.shiftSaveQueueAction(3))); }); test("SaveSaga should set the correct version numbers if the save queue was compacted", (t) => { const saveQueue = createSaveQueueFromUpdateActions( [ - [UpdateActions.updateSkeletonTracing(initialState, [1, 2, 3], [0, 0, 1], 1)], - [UpdateActions.updateSkeletonTracing(initialState, [2, 3, 4], [0, 0, 1], 2)], - [UpdateActions.updateSkeletonTracing(initialState, [3, 4, 5], [0, 0, 1], 3)], + [UpdateActions.updateSkeletonTracing(initialState.tracing, [1, 2, 3], [], [0, 0, 1], 1)], + [UpdateActions.updateSkeletonTracing(initialState.tracing, [2, 3, 4], [], [0, 0, 1], 2)], + [UpdateActions.updateSkeletonTracing(initialState.tracing, [3, 4, 5], [], [0, 0, 1], 3)], ], TIMESTAMP, tracingId, ); - const saga = sendSaveRequestToServer(TRACING_TYPE, tracingId); + const saga = sendSaveRequestToServer(); saga.next(); saga.next(saveQueue); - saga.next({ - version: LAST_VERSION, - type: TRACING_TYPE, - }); + saga.next([ + { + version: LAST_VERSION, + type: TRACING_TYPE, + tracingId, + }, + ]); + saga.next(annotationId); saga.next(TRACINGSTORE_URL); // two of the updateTracing update actions are removed by compactSaveQueue expectValueDeepEqual( @@ -338,11 +368,7 @@ test("SaveSaga should set the correct version numbers if the save queue was comp put(SaveActions.setVersionNumberAction(LAST_VERSION + 1, TRACING_TYPE, tracingId)), ); expectValueDeepEqual(t, saga.next(), put(SaveActions.setLastSaveTimestampAction())); - expectValueDeepEqual( - t, - saga.next(), - put(SaveActions.shiftSaveQueueAction(3, TRACING_TYPE, tracingId)), - ); + expectValueDeepEqual(t, saga.next(), put(SaveActions.shiftSaveQueueAction(3))); }); test("SaveSaga addVersionNumbers should set the correct version numbers", (t) => { const saveQueue = createSaveQueueFromUpdateActions( From f5bda6d115482ee54e529caa316b36c0a181e9c1 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Mon, 16 Sep 2024 15:50:21 +0200 Subject: [PATCH 055/150] fix volumetracing_saga.spec.ts --- .../test/sagas/volumetracing/volumetracing_saga.spec.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/frontend/javascripts/test/sagas/volumetracing/volumetracing_saga.spec.ts b/frontend/javascripts/test/sagas/volumetracing/volumetracing_saga.spec.ts index d7b16773e50..f30a50ea181 100644 --- a/frontend/javascripts/test/sagas/volumetracing/volumetracing_saga.spec.ts +++ b/frontend/javascripts/test/sagas/volumetracing/volumetracing_saga.spec.ts @@ -144,7 +144,6 @@ test("VolumeTracingSaga shouldn't do anything if unchanged (saga test)", (t) => const saga = setupSavingForTracingType( VolumeTracingActions.initializeVolumeTracingAction(serverVolumeTracing), ); - saga.next(); // forking pushSaveQueueAsync saga.next(); saga.next(initialState.tracing.volumes[0]); @@ -165,7 +164,6 @@ test("VolumeTracingSaga should do something if changed (saga test)", (t) => { const saga = setupSavingForTracingType( VolumeTracingActions.initializeVolumeTracingAction(serverVolumeTracing), ); - saga.next(); // forking pushSaveQueueAsync saga.next(); saga.next(initialState.tracing.volumes[0]); From 23f64d9ff11bc82925a4afd703d1e6baf88c6e35 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Mon, 16 Sep 2024 15:56:46 +0200 Subject: [PATCH 056/150] fix skeletontracing saga spec --- .../test/sagas/skeletontracing_saga.spec.ts | 133 ++++++++++-------- 1 file changed, 76 insertions(+), 57 deletions(-) diff --git a/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts b/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts index f612805bcc7..b19c883f296 100644 --- a/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts +++ b/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts @@ -27,7 +27,8 @@ import { TreeTypeEnum } from "oxalis/constants"; import type { Action } from "oxalis/model/actions/actions"; import type { ServerSkeletonTracing } from "types/api_flow_types"; import { enforceSkeletonTracing } from "oxalis/model/accessors/skeletontracing_accessor"; -import { addTracingIdToActions } from "oxalis/model/reducers/save_reducer"; +import { UpdateAction } from "oxalis/model/sagas/update_actions"; +import { TracingStats } from "oxalis/model/accessors/annotation_accessor"; const TIMESTAMP = 1494347146379; const DateMock = { @@ -94,6 +95,22 @@ function compactSaveQueueWithUpdateActions( ); } +function createCompactedSaveQueueFromUpdateActions( + updateActions: UpdateAction[][], + timestamp: number, + tracing: SkeletonTracing, + stats: TracingStats | null = null, +) { + return compactSaveQueue( + createSaveQueueFromUpdateActions( + updateActions.map((batch) => compactUpdateActions(batch, tracing)), + timestamp, + tracing.tracingId, + stats, + ), + ); +} + const skeletonTracing: SkeletonTracing = { type: "skeleton", createdTimestamp: 0, @@ -187,7 +204,6 @@ test("SkeletonTracingSaga shouldn't do anything if unchanged (saga test)", (t) = const saga = setupSavingForTracingType( SkeletonTracingActions.initializeSkeletonTracingAction(serverSkeletonTracing), ); - saga.next(); // forking pushSaveQueueAsync saga.next(); saga.next(initialState.tracing.skeleton); @@ -207,7 +223,6 @@ test("SkeletonTracingSaga should do something if changed (saga test)", (t) => { const saga = setupSavingForTracingType( SkeletonTracingActions.initializeSkeletonTracingAction(serverSkeletonTracing), ); - saga.next(); // forking pushSaveQueueAsync saga.next(); saga.next(initialState.tracing.skeleton); @@ -642,20 +657,18 @@ test("compactUpdateActions should detect a tree merge (1/3)", (t) => { testState.flycam, newState.flycam, ); - const saveQueue = createSaveQueueFromUpdateActions( + const simplifiedUpdateActions = createCompactedSaveQueueFromUpdateActions( [updateActions], TIMESTAMP, - skeletonTracing.tracingId, - ); - const simplifiedUpdateActions = compactSaveQueueWithUpdateActions( - saveQueue, - enforceSkeletonTracing(newState.tracing), + skeletonTracing, ); + const simplifiedFirstBatch = simplifiedUpdateActions[0].actions; // This should result in a moved treeComponent of size three t.deepEqual(simplifiedFirstBatch[0], { name: "moveTreeComponent", value: { + actionTracingId: "tracingId", sourceId: 1, targetId: 2, nodeIds: [1, 2, 3], @@ -665,6 +678,7 @@ test("compactUpdateActions should detect a tree merge (1/3)", (t) => { t.deepEqual(simplifiedFirstBatch[1], { name: "deleteTree", value: { + actionTracingId: "tracingId", id: 1, }, }); @@ -672,6 +686,7 @@ test("compactUpdateActions should detect a tree merge (1/3)", (t) => { t.deepEqual(simplifiedFirstBatch[2], { name: "createEdge", value: { + actionTracingId: "tracingId", treeId: 2, source: 4, target: 1, @@ -706,20 +721,18 @@ test("compactUpdateActions should detect a tree merge (2/3)", (t) => { testDiffing(newState1.tracing, newState2.tracing, newState1.flycam, newState2.flycam), ); // compactUpdateActions is triggered by the saving, it can therefore contain the results of more than one diffing - const saveQueue = createSaveQueueFromUpdateActions( + const simplifiedUpdateActions = createCompactedSaveQueueFromUpdateActions( updateActions, TIMESTAMP, - skeletonTracing.tracingId, - ); - const simplifiedUpdateActions = compactSaveQueueWithUpdateActions( - saveQueue, - enforceSkeletonTracing(newState2.tracing), + skeletonTracing, ); + // This should result in one created node and its edge (a) const simplifiedFirstBatch = simplifiedUpdateActions[0].actions; t.like(simplifiedFirstBatch[0], { name: "createNode", value: { + actionTracingId: "tracingId", id: 5, treeId: 2, }, @@ -727,6 +740,7 @@ test("compactUpdateActions should detect a tree merge (2/3)", (t) => { t.like(simplifiedFirstBatch[1], { name: "createEdge", value: { + actionTracingId: "tracingId", treeId: 2, source: 4, target: 5, @@ -738,6 +752,7 @@ test("compactUpdateActions should detect a tree merge (2/3)", (t) => { t.deepEqual(simplifiedSecondBatch[0], { name: "moveTreeComponent", value: { + actionTracingId: "tracingId", sourceId: 1, targetId: 2, nodeIds: [1, 2, 3], @@ -747,6 +762,7 @@ test("compactUpdateActions should detect a tree merge (2/3)", (t) => { t.deepEqual(simplifiedSecondBatch[1], { name: "deleteTree", value: { + actionTracingId: "tracingId", id: 1, }, }); @@ -757,6 +773,7 @@ test("compactUpdateActions should detect a tree merge (2/3)", (t) => { t.deepEqual(simplifiedSecondBatch[4], { name: "createEdge", value: { + actionTracingId: "tracingId", treeId: 2, source: 5, target: 1, @@ -812,20 +829,17 @@ test("compactUpdateActions should detect a tree merge (3/3)", (t) => { ), ); // compactUpdateActions is triggered by the saving, it can therefore contain the results of more than one diffing - const saveQueue = createSaveQueueFromUpdateActions( + const simplifiedUpdateActions = createCompactedSaveQueueFromUpdateActions( updateActions, TIMESTAMP, - skeletonTracing.tracingId, - ); - const simplifiedUpdateActions = compactSaveQueueWithUpdateActions( - saveQueue, - enforceSkeletonTracing(newState.tracing), + skeletonTracing, ); // This should result in a moved treeComponent of size one (a) const simplifiedFirstBatch = simplifiedUpdateActions[0].actions; t.deepEqual(simplifiedFirstBatch[0], { name: "moveTreeComponent", value: { + actionTracingId: "tracingId", sourceId: 2, targetId: 1, nodeIds: [4], @@ -835,6 +849,7 @@ test("compactUpdateActions should detect a tree merge (3/3)", (t) => { t.deepEqual(simplifiedFirstBatch[1], { name: "deleteTree", value: { + actionTracingId: "tracingId", id: 2, }, }); @@ -842,6 +857,7 @@ test("compactUpdateActions should detect a tree merge (3/3)", (t) => { t.deepEqual(simplifiedFirstBatch[2], { name: "createEdge", value: { + actionTracingId: "tracingId", treeId: 1, source: 1, target: 4, @@ -860,6 +876,7 @@ test("compactUpdateActions should detect a tree merge (3/3)", (t) => { t.deepEqual(simplifiedThirdBatch[0], { name: "moveTreeComponent", value: { + actionTracingId: "tracingId", sourceId: 2, targetId: 1, nodeIds: [5, 6], @@ -868,12 +885,14 @@ test("compactUpdateActions should detect a tree merge (3/3)", (t) => { t.deepEqual(simplifiedThirdBatch[1], { name: "deleteTree", value: { + actionTracingId: "tracingId", id: 2, }, }); t.deepEqual(simplifiedThirdBatch[2], { name: "createEdge", value: { + actionTracingId: "tracingId", treeId: 1, source: 1, target: 6, @@ -898,20 +917,19 @@ test("compactUpdateActions should detect a tree split (1/3)", (t) => { testState.flycam, newState.flycam, ); - const saveQueue = createSaveQueueFromUpdateActions( + + const simplifiedUpdateActions = createCompactedSaveQueueFromUpdateActions( [updateActions], TIMESTAMP, - skeletonTracing.tracingId, - ); - const simplifiedUpdateActions = compactSaveQueueWithUpdateActions( - saveQueue, - enforceSkeletonTracing(newState.tracing), + skeletonTracing, ); + // This should result in a new tree const simplifiedFirstBatch = simplifiedUpdateActions[0].actions; t.like(simplifiedFirstBatch[0], { name: "createTree", value: { + actionTracingId: "tracingId", id: 2, }, }); @@ -919,6 +937,7 @@ test("compactUpdateActions should detect a tree split (1/3)", (t) => { t.deepEqual(simplifiedFirstBatch[1], { name: "moveTreeComponent", value: { + actionTracingId: "tracingId", sourceId: 1, targetId: 2, nodeIds: [3, 4], @@ -928,6 +947,7 @@ test("compactUpdateActions should detect a tree split (1/3)", (t) => { t.deepEqual(simplifiedFirstBatch[2], { name: "deleteNode", value: { + actionTracingId: "tracingId", nodeId: 2, treeId: 1, }, @@ -960,26 +980,24 @@ test("compactUpdateActions should detect a tree split (2/3)", (t) => { testState.flycam, newState.flycam, ); - const saveQueue = createSaveQueueFromUpdateActions( + const simplifiedUpdateActions = createCompactedSaveQueueFromUpdateActions( [updateActions], TIMESTAMP, - skeletonTracing.tracingId, - ); - const simplifiedUpdateActions = compactSaveQueueWithUpdateActions( - saveQueue, - enforceSkeletonTracing(newState.tracing), + skeletonTracing, ); // This should result in two new trees and two moved treeComponents of size three and two const simplifiedFirstBatch = simplifiedUpdateActions[0].actions; t.like(simplifiedFirstBatch[0], { name: "createTree", value: { + actionTracingId: "tracingId", id: 2, }, }); t.deepEqual(simplifiedFirstBatch[1], { name: "moveTreeComponent", value: { + actionTracingId: "tracingId", sourceId: 1, targetId: 2, nodeIds: [3, 4], @@ -988,12 +1006,14 @@ test("compactUpdateActions should detect a tree split (2/3)", (t) => { t.like(simplifiedFirstBatch[2], { name: "createTree", value: { + actionTracingId: "tracingId", id: 3, }, }); t.deepEqual(simplifiedFirstBatch[3], { name: "moveTreeComponent", value: { + actionTracingId: "tracingId", sourceId: 1, targetId: 3, nodeIds: [5, 6, 7], @@ -1003,6 +1023,7 @@ test("compactUpdateActions should detect a tree split (2/3)", (t) => { t.deepEqual(simplifiedFirstBatch[4], { name: "deleteNode", value: { + actionTracingId: "tracingId", nodeId: 2, treeId: 1, }, @@ -1036,20 +1057,17 @@ test("compactUpdateActions should detect a tree split (3/3)", (t) => { updateActions.push( testDiffing(newState1.tracing, newState2.tracing, newState1.flycam, newState2.flycam), ); - const saveQueue = createSaveQueueFromUpdateActions( + const simplifiedUpdateActions = createCompactedSaveQueueFromUpdateActions( updateActions, TIMESTAMP, - skeletonTracing.tracingId, - ); - const simplifiedUpdateActions = compactSaveQueueWithUpdateActions( - saveQueue, - enforceSkeletonTracing(newState2.tracing), + skeletonTracing, ); // This should result in the creation of a new tree (a) const simplifiedFirstBatch = simplifiedUpdateActions[0].actions; t.like(simplifiedFirstBatch[0], { name: "createTree", value: { + actionTracingId: "tracingId", id: 2, }, }); @@ -1057,6 +1075,7 @@ test("compactUpdateActions should detect a tree split (3/3)", (t) => { t.deepEqual(simplifiedFirstBatch[1], { name: "moveTreeComponent", value: { + actionTracingId: "tracingId", sourceId: 1, targetId: 2, nodeIds: [3, 4, 5, 6], @@ -1066,6 +1085,7 @@ test("compactUpdateActions should detect a tree split (3/3)", (t) => { t.deepEqual(simplifiedFirstBatch[2], { name: "deleteNode", value: { + actionTracingId: "tracingId", nodeId: 2, treeId: 1, }, @@ -1078,6 +1098,7 @@ test("compactUpdateActions should detect a tree split (3/3)", (t) => { t.like(simplifiedSecondBatch[0], { name: "createTree", value: { + actionTracingId: "tracingId", id: 3, }, }); @@ -1085,6 +1106,7 @@ test("compactUpdateActions should detect a tree split (3/3)", (t) => { t.deepEqual(simplifiedSecondBatch[1], { name: "moveTreeComponent", value: { + actionTracingId: "tracingId", sourceId: 2, targetId: 3, nodeIds: [5, 6], @@ -1094,6 +1116,7 @@ test("compactUpdateActions should detect a tree split (3/3)", (t) => { t.deepEqual(simplifiedSecondBatch[2], { name: "deleteNode", value: { + actionTracingId: "tracingId", nodeId: 4, treeId: 2, }, @@ -1127,21 +1150,22 @@ test("compactUpdateActions should do nothing if it cannot compact", (t) => { testState.flycam, newState.flycam, ); - const saveQueue = createSaveQueueFromUpdateActions( + const saveQueueOriginal = createSaveQueueFromUpdateActions( [updateActions], TIMESTAMP, skeletonTracing.tracingId, ); - const simplifiedUpdateActions = compactSaveQueueWithUpdateActions( - saveQueue, - enforceSkeletonTracing(newState.tracing), + const simplifiedUpdateActions = createCompactedSaveQueueFromUpdateActions( + [updateActions], + TIMESTAMP, + skeletonTracing, ); // The deleteTree optimization in compactUpdateActions (that is unrelated to this test) // will remove the first deleteNode update action as the first tree is deleted because of the merge, // therefore remove it here as well - saveQueue[0].actions.shift(); + saveQueueOriginal[0].actions.shift(); // Nothing should be changed as the moveTreeComponent update action cannot be inserted - t.deepEqual(simplifiedUpdateActions, saveQueue); + t.deepEqual(simplifiedUpdateActions, saveQueueOriginal); }); test("compactUpdateActions should detect a deleted tree", (t) => { const testState = ChainReducer(initialState) @@ -1160,19 +1184,16 @@ test("compactUpdateActions should detect a deleted tree", (t) => { testState.flycam, newState.flycam, ); - const saveQueue = createSaveQueueFromUpdateActions( + const simplifiedUpdateActions = createCompactedSaveQueueFromUpdateActions( [updateActions], TIMESTAMP, - skeletonTracing.tracingId, - ); - const simplifiedUpdateActions = compactSaveQueueWithUpdateActions( - saveQueue, - enforceSkeletonTracing(newState.tracing), + skeletonTracing, ); const simplifiedFirstBatch = simplifiedUpdateActions[0].actions; t.deepEqual(simplifiedFirstBatch[0], { name: "deleteTree", value: { + actionTracingId: "tracingId", id: 2, }, }); @@ -1196,19 +1217,16 @@ test("compactUpdateActions should not detect a deleted tree if there is no delet testState.flycam, newState.flycam, ); - const saveQueue = createSaveQueueFromUpdateActions( + const simplifiedUpdateActions = createCompactedSaveQueueFromUpdateActions( [updateActions], TIMESTAMP, - skeletonTracing.tracingId, - ); - const simplifiedUpdateActions = compactSaveQueueWithUpdateActions( - saveQueue, - enforceSkeletonTracing(newState.tracing), + skeletonTracing, ); const simplifiedFirstBatch = simplifiedUpdateActions[0].actions; t.deepEqual(simplifiedFirstBatch[0], { name: "deleteNode", value: { + actionTracingId: "tracingId", nodeId: 2, treeId: 2, }, @@ -1216,6 +1234,7 @@ test("compactUpdateActions should not detect a deleted tree if there is no delet t.deepEqual(simplifiedFirstBatch[1], { name: "deleteNode", value: { + actionTracingId: "tracingId", nodeId: 3, treeId: 2, }, From 93420bf4788af950d8a4f2d6e2351fa8140304b1 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Mon, 16 Sep 2024 16:01:38 +0200 Subject: [PATCH 057/150] fix saga_integration spec --- frontend/javascripts/test/sagas/saga_integration.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/javascripts/test/sagas/saga_integration.spec.ts b/frontend/javascripts/test/sagas/saga_integration.spec.ts index 33d818926c9..1d137bd72b7 100644 --- a/frontend/javascripts/test/sagas/saga_integration.spec.ts +++ b/frontend/javascripts/test/sagas/saga_integration.spec.ts @@ -14,6 +14,7 @@ import { setActiveUserAction } from "oxalis/model/actions/user_actions"; import dummyUser from "test/fixtures/dummy_user"; import { hasRootSagaCrashed } from "oxalis/model/sagas/root_saga"; import { omit } from "lodash"; +import { tracing as TaskTracing } from "test/fixtures/tasktracing_server_objects"; const { createTreeMapFromTreeArray, @@ -67,7 +68,7 @@ test.serial( ], ], TIMESTAMP, - "tracingId", + TaskTracing.id, getStats(state.tracing, "skeleton", "irrelevant_in_skeleton_case") || undefined, ); // Reset the info field which is just for debugging purposes From d96a001843c719fea1149784f496f6e2336ddb21 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Mon, 16 Sep 2024 16:13:52 +0200 Subject: [PATCH 058/150] improve typing in saga integration spec --- .../test/sagas/saga_integration.spec.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/frontend/javascripts/test/sagas/saga_integration.spec.ts b/frontend/javascripts/test/sagas/saga_integration.spec.ts index 1d137bd72b7..62031c20d51 100644 --- a/frontend/javascripts/test/sagas/saga_integration.spec.ts +++ b/frontend/javascripts/test/sagas/saga_integration.spec.ts @@ -16,16 +16,18 @@ import { hasRootSagaCrashed } from "oxalis/model/sagas/root_saga"; import { omit } from "lodash"; import { tracing as TaskTracing } from "test/fixtures/tasktracing_server_objects"; -const { - createTreeMapFromTreeArray, - generateTreeName, -} = require("oxalis/model/reducers/skeletontracing_reducer_helpers"); +const { createTreeMapFromTreeArray, generateTreeName } = + require("oxalis/model/reducers/skeletontracing_reducer_helpers") as typeof import("oxalis/model/reducers/skeletontracing_reducer_helpers"); const { addTreesAndGroupsAction, deleteNodeAction } = mockRequire.reRequire( "oxalis/model/actions/skeletontracing_actions", -); -const { discardSaveQueuesAction } = mockRequire.reRequire("oxalis/model/actions/save_actions"); -const UpdateActions = mockRequire.reRequire("oxalis/model/sagas/update_actions"); +) as typeof import("oxalis/model/actions/skeletontracing_actions"); +const { discardSaveQueuesAction } = mockRequire.reRequire( + "oxalis/model/actions/save_actions", +) as typeof import("oxalis/model/actions/save_actions"); +const UpdateActions = mockRequire.reRequire( + "oxalis/model/sagas/update_actions", +) as typeof import("oxalis/model/sagas/update_actions"); test.beforeEach(async (t) => { // Setup oxalis, this will execute model.fetch(...) and initialize the store with the tracing, etc. @@ -59,7 +61,7 @@ test.serial( [ UpdateActions.updateTree(treeWithCorrectName), UpdateActions.updateSkeletonTracing( - Store.getState().tracing.skeleton, + enforceSkeletonTracing(Store.getState().tracing), [1, 2, 3], [], [0, 0, 0], From 9234507156aedcdc483baa0e155817b8ce66a76b Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 17 Sep 2024 09:35:32 +0200 Subject: [PATCH 059/150] update action log --- frontend/javascripts/admin/admin_rest_api.ts | 10 ++++---- .../oxalis/model/sagas/save_saga.ts | 10 +++----- .../javascripts/oxalis/view/version_list.tsx | 20 +++++----------- .../test/sagas/skeletontracing_saga.spec.ts | 4 ++-- .../controllers/TSAnnotationController.scala | 13 ++++++++++ .../controllers/TracingController.scala | 9 ------- ...alableminds.webknossos.tracingstore.routes | 24 ++++++++----------- 7 files changed, 38 insertions(+), 52 deletions(-) diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts index 274ca537bb1..e3d69154408 100644 --- a/frontend/javascripts/admin/admin_rest_api.ts +++ b/frontend/javascripts/admin/admin_rest_api.ts @@ -927,8 +927,7 @@ export async function getTracingForAnnotationType( export function getUpdateActionLog( tracingStoreUrl: string, - tracingId: string, - versionedObjectType: SaveQueueType, + annotationId: string, oldestVersion?: number, newestVersion?: number, ): Promise> { @@ -942,19 +941,18 @@ export function getUpdateActionLog( params.append("newestVersion", newestVersion.toString()); } return Request.receiveJSON( - `${tracingStoreUrl}/tracings/${versionedObjectType}/${tracingId}/updateActionLog?${params}`, + `${tracingStoreUrl}/tracings/annotation/${annotationId}/updateActionLog?${params}`, ); }); } export function getNewestVersionForTracing( tracingStoreUrl: string, - tracingId: string, - tracingType: SaveQueueType, + annotationId: string, ): Promise { return doWithToken((token) => Request.receiveJSON( - `${tracingStoreUrl}/tracings/${tracingType}/${tracingId}/newestVersion?token=${token}`, + `${tracingStoreUrl}/tracings/annotation/${annotationId}/newestVersion?token=${token}`, ).then((obj) => obj.version), ); } diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts index d9253fc6606..508a62a8e99 100644 --- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts @@ -488,20 +488,16 @@ function* watchForSaveConflicts() { const maybeSkeletonTracing = yield* select((state) => state.tracing.skeleton); const volumeTracings = yield* select((state) => state.tracing.volumes); const tracingStoreUrl = yield* select((state) => state.tracing.tracingStore.url); + const annotationId = yield* select((state) => state.tracing.annotationId); const tracings: Array = _.compact([ ...volumeTracings, maybeSkeletonTracing, ]); - for (const tracing of tracings) { - const versionOnServer = yield* call( - getNewestVersionForTracing, - tracingStoreUrl, - tracing.tracingId, - tracing.type, - ); + const versionOnServer = yield* call(getNewestVersionForTracing, tracingStoreUrl, annotationId); + for (const tracing of tracings) { // Read the tracing version again from the store, since the // old reference to tracing might be outdated now due to the // immutability. diff --git a/frontend/javascripts/oxalis/view/version_list.tsx b/frontend/javascripts/oxalis/view/version_list.tsx index 9ccdaac3bf5..cb5d79183cf 100644 --- a/frontend/javascripts/oxalis/view/version_list.tsx +++ b/frontend/javascripts/oxalis/view/version_list.tsx @@ -148,8 +148,7 @@ const getGroupedAndChunkedVersions = _.memoize( async function getUpdateActionLogPage( props: Props, tracingStoreUrl: string, - tracingId: string, - versionedObjectType: SaveQueueType, + annotationId: string, newestVersion: number, // 0 is the "newest" page (i.e., the page in which the newest version is) relativePageNumber: number, @@ -177,8 +176,7 @@ async function getUpdateActionLogPage( const updateActionLog = await getUpdateActionLog( tracingStoreUrl, - tracingId, - versionedObjectType, + annotationId, oldestVersionInPage, newestVersionInPage, ); @@ -203,9 +201,10 @@ async function getUpdateActionLogPage( function VersionList(props: Props) { const { tracing } = props; const tracingStoreUrl = useSelector((state: OxalisState) => state.tracing.tracingStore.url); + const annotationId = useSelector((state: OxalisState) => state.tracing.annotationId); const newestVersion = useFetch( - () => getNewestVersionForTracing(tracingStoreUrl, tracing.tracingId, props.versionedObjectType), + () => getNewestVersionForTracing(tracingStoreUrl, annotationId), null, [tracing], ); @@ -233,17 +232,10 @@ function InnerVersionList(props: Props & { newestVersion: number }) { if (pageParam == null) { pageParam = Math.floor((newestVersion - initialVersion) / ENTRIES_PER_PAGE); } - const { tracingId } = props.tracing; const { url: tracingStoreUrl } = Store.getState().tracing.tracingStore; + const annotationId = Store.getState().tracing.annotationId; - return getUpdateActionLogPage( - props, - tracingStoreUrl, - tracingId, - props.versionedObjectType, - newestVersion, - pageParam, - ); + return getUpdateActionLogPage(props, tracingStoreUrl, annotationId, newestVersion, pageParam); } const queryKey = ["versions", props.tracing.tracingId]; diff --git a/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts b/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts index b19c883f296..7f5ab13a304 100644 --- a/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts +++ b/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts @@ -27,8 +27,8 @@ import { TreeTypeEnum } from "oxalis/constants"; import type { Action } from "oxalis/model/actions/actions"; import type { ServerSkeletonTracing } from "types/api_flow_types"; import { enforceSkeletonTracing } from "oxalis/model/accessors/skeletontracing_accessor"; -import { UpdateAction } from "oxalis/model/sagas/update_actions"; -import { TracingStats } from "oxalis/model/accessors/annotation_accessor"; +import type { UpdateAction } from "oxalis/model/sagas/update_actions"; +import type { TracingStats } from "oxalis/model/accessors/annotation_accessor"; const TIMESTAMP = 1494347146379; const DateMock = { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index 2245180a963..e2858994667 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -12,6 +12,7 @@ import com.scalableminds.webknossos.tracingstore.annotation.{ UpdateActionGroup } import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService +import play.api.libs.json.Json import play.api.mvc.{Action, AnyContent, PlayBodyParsers} import scala.concurrent.ExecutionContext @@ -68,6 +69,18 @@ class TSAnnotationController @Inject()( } } + def newestVersion(token: Option[String], annotationId: String): Action[AnyContent] = Action.async { + implicit request => + log() { + accessTokenService.validateAccess(UserAccessRequest.readAnnotation(annotationId), + urlOrHeaderToken(token, request)) { + for { + newestVersion <- annotationService.currentMaterializableVersion(annotationId) + } yield JsonOk(Json.obj("version" -> newestVersion)) + } + } + } + def updateActionStatistics(token: Option[String], tracingId: String): Action[AnyContent] = Action.async { implicit request => log() { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala index a2db158e6a6..d39cf16c671 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala @@ -103,15 +103,6 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C } } - def newestVersion(token: Option[String], annotationId: String, tracingId: String): Action[AnyContent] = Action.async { - implicit request => - log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), token) { - Fox.successful(JsonOk(Json.obj("version" -> 0L))) // TODO remove in favor of annotation-wide - } - } - } - def mergedFromIds(token: Option[String], persist: Boolean): Action[List[Option[TracingSelector]]] = Action.async(validateJson[List[Option[TracingSelector]]]) { implicit request => log() { diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 9058706cf74..1c7a3c5ab00 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -3,20 +3,20 @@ # ~~~~ # Health endpoint -GET /health @com.scalableminds.webknossos.tracingstore.controllers.Application.health +GET /health @com.scalableminds.webknossos.tracingstore.controllers.Application.health -POST /annotation/save @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.save(token: Option[String], annotationId: String) -GET /annotation/:annotationId @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.get(token: Option[String], annotationId: String, version: Option[Long]) -POST /annotation/:annotationId/update @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.update(token: Option[String], annotationId: String) -POST /annotation/:annotationId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionLog(token: Option[String], annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]) -GET /annotation/:annotationId/updateActionStatistics @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionStatistics(token: Option[String], annotationId: String) +POST /annotation/save @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.save(token: Option[String], annotationId: String) +GET /annotation/:annotationId @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.get(token: Option[String], annotationId: String, version: Option[Long]) +POST /annotation/:annotationId/update @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.update(token: Option[String], annotationId: String) +GET /annotation/:annotationId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionLog(token: Option[String], annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]) +GET /annotation/:annotationId/updateActionStatistics @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionStatistics(token: Option[String], annotationId: String) +GET /annotation/:annotationId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.newestVersion(token: Option[String], annotationId: String) # Volume tracings POST /volume/save @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.save(token: Option[String]) POST /volume/:annotationId/:tracingId/initialData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.initialData(token: Option[String], annotationId: String, tracingId: String, minResolution: Option[Int], maxResolution: Option[Int]) POST /volume/:annotationId/:tracingId/initialDataMultiple @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.initialDataMultiple(token: Option[String], annotationId: String, tracingId: String) GET /volume/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.get(token: Option[String], annotationId: String, tracingId: String, version: Option[Long]) -GET /volume/:annotationId/:tracingId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.newestVersion(token: Option[String], annotationId: String, tracingId: String) GET /volume/:annotationId/:tracingId/allDataZip @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.allDataZip(token: Option[String], annotationId: String, tracingId: String, volumeDataZipFormat: String, version: Option[Long], voxelSize: Option[String], voxelSizeUnit: Option[String]) POST /volume/:annotationId/:tracingId/data @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.data(token: Option[String], annotationId: String, tracingId: String) POST /volume/:annotationId/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.duplicate(token: Option[String], annotationId: String, tracingId: String, fromTask: Option[Boolean], minResolution: Option[Int], maxResolution: Option[Int], downsample: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) @@ -71,12 +71,8 @@ GET /volume/zarr3_experimental/:annotationId/:tracingId/:mag/:coordinate # Skeleton tracings POST /skeleton/save @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.save(token: Option[String]) POST /skeleton/saveMultiple @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.saveMultiple(token: Option[String]) - POST /skeleton/mergedFromContents @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.mergedFromContents(token: Option[String], persist: Boolean) POST /skeleton/mergedFromIds @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.mergedFromIds(token: Option[String], persist: Boolean) - -GET /skeleton/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.get(token: Option[String], annotationId: String, tracingId: String, version: Option[Long]) -GET /skeleton/:annotationId/:tracingId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.newestVersion(token: Option[String], annotationId: String, tracingId: String) -POST /skeleton/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.getMultiple(token: Option[String]) - -POST /skeleton/:annotationId/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.duplicate(token: Option[String], annotationId: String, tracingId: String, version: Option[Long], fromTask: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) +GET /skeleton/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.get(token: Option[String], annotationId: String, tracingId: String, version: Option[Long]) +POST /skeleton/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.getMultiple(token: Option[String]) +POST /skeleton/:annotationId/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.duplicate(token: Option[String], annotationId: String, tracingId: String, version: Option[Long], fromTask: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) From a42fe4701bbd221406cc45a2165bf1fd81585612 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 17 Sep 2024 12:08:39 +0200 Subject: [PATCH 060/150] wip editable mappings. tokenContext. --- frontend/javascripts/admin/admin_rest_api.ts | 6 +- .../oxalis/model/sagas/proofread_saga.ts | 3 +- .../oxalis/model_initialization.ts | 4 +- .../util/accesscontext/TokenContext.scala | 4 + .../util/mvc/ExtendedController.scala | 5 +- .../controllers/BinaryDataController.scala | 27 +-- .../controllers/DSMeshController.scala | 21 +- .../controllers/DataSourceController.scala | 146 +++++------- .../controllers/ExportsController.scala | 2 +- .../controllers/ZarrStreamingController.scala | 104 ++++----- .../webknossos/datastore/rpc/RPCRequest.scala | 4 + .../services/AccessTokenService.scala | 26 +-- .../services/DSFullMeshService.scala | 27 ++- .../services/DSRemoteTracingstoreClient.scala | 81 +++---- .../services/DSRemoteWebknossosClient.scala | 32 ++- .../services/MeshMappingHelper.scala | 6 +- .../services/uploading/ComposeService.scala | 5 +- .../TSRemoteDatastoreClient.scala | 89 +++----- .../TSRemoteWebknossosClient.scala | 5 +- .../AnnotationTransactionService.scala | 41 ++-- .../annotation/AnnotationWithTracings.scala | 5 + .../annotation/TSAnnotationService.scala | 45 ++-- .../SkeletonTracingController.scala | 10 +- .../controllers/TSAnnotationController.scala | 22 +- .../controllers/TracingController.scala | 33 +-- .../controllers/VolumeTracingController.scala | 215 +++++++----------- ...VolumeTracingZarrStreamingController.scala | 79 +++---- .../tracings/RemoteFallbackLayer.scala | 12 +- .../tracings/TracingService.scala | 27 +-- .../EditableMappingLayer.scala | 25 +- .../EditableMappingService.scala | 204 +++++++---------- .../EditableMappingUpdater.scala | 28 +-- .../skeleton/SkeletonTracingService.scala | 13 +- .../tracings/volume/TSFullMeshService.scala | 71 +++--- .../volume/VolumeSegmentIndexBuffer.scala | 19 +- .../volume/VolumeSegmentIndexService.scala | 68 +++--- .../VolumeSegmentStatisticsService.scala | 57 +++-- .../volume/VolumeTracingBucketHelper.scala | 14 +- .../volume/VolumeTracingDownsampling.scala | 21 +- .../tracings/volume/VolumeTracingLayer.scala | 3 +- .../volume/VolumeTracingService.scala | 182 +++++++-------- ...alableminds.webknossos.tracingstore.routes | 1 + 42 files changed, 746 insertions(+), 1046 deletions(-) create mode 100644 util/src/main/scala/com/scalableminds/util/accesscontext/TokenContext.scala diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts index e3d69154408..ee6816d8845 100644 --- a/frontend/javascripts/admin/admin_rest_api.ts +++ b/frontend/javascripts/admin/admin_rest_api.ts @@ -1614,11 +1614,12 @@ export function fetchMapping( export function makeMappingEditable( tracingStoreUrl: string, + annotationId: string, tracingId: string, ): Promise { return doWithToken((token) => Request.receiveJSON( - `${tracingStoreUrl}/tracings/volume/${tracingId}/makeMappingEditable?token=${token}`, + `${tracingStoreUrl}/tracings/volume/${annotationId}/${tracingId}/makeMappingEditable?token=${token}`, { method: "POST", }, @@ -1628,10 +1629,11 @@ export function makeMappingEditable( export function getEditableMappingInfo( tracingStoreUrl: string, + annotationId: string, tracingId: string, ): Promise { return doWithToken((token) => - Request.receiveJSON(`${tracingStoreUrl}/tracings/mapping/${tracingId}/info?token=${token}`), + Request.receiveJSON(`${tracingStoreUrl}/tracings/mapping/${annotationId}/${tracingId}/info?token=${token}`), ); } diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts index e70b43a7058..0fc1a9e5d78 100644 --- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts @@ -265,6 +265,7 @@ function* createEditableMapping(): Saga { * name of the HDF5 mapping for which the editable mapping is about to be created. */ const tracingStoreUrl = yield* select((state) => state.tracing.tracingStore.url); + const annotationId = yield* select((state) => state.tracing.annotationId); // Save before making the mapping editable to make sure the correct mapping is activated in the backend yield* call([Model, Model.ensureSavedState]); // Get volume tracing again to make sure the version is up to date @@ -275,7 +276,7 @@ function* createEditableMapping(): Saga { const volumeTracingId = upToDateVolumeTracing.tracingId; const layerName = volumeTracingId; - const serverEditableMapping = yield* call(makeMappingEditable, tracingStoreUrl, volumeTracingId); + const serverEditableMapping = yield* call(makeMappingEditable, tracingStoreUrl, annotationId, volumeTracingId); // The server increments the volume tracing's version by 1 when switching the mapping to an editable one yield* put(setVersionNumberAction(upToDateVolumeTracing.version + 1, "volume", volumeTracingId)); yield* put(setMappingNameAction(layerName, serverEditableMapping.mappingName, "HDF5")); diff --git a/frontend/javascripts/oxalis/model_initialization.ts b/frontend/javascripts/oxalis/model_initialization.ts index c87d88844a0..c9ebfe4a751 100644 --- a/frontend/javascripts/oxalis/model_initialization.ts +++ b/frontend/javascripts/oxalis/model_initialization.ts @@ -213,6 +213,7 @@ export async function initialize( if (annotation != null) { const editableMappings = await fetchEditableMappings( annotation.tracingStore.url, + annotation.id, serverVolumeTracings, ); initializeTracing(annotation, serverTracings, editableMappings); @@ -248,11 +249,12 @@ async function fetchParallel( async function fetchEditableMappings( tracingStoreUrl: string, + annotationId: string, serverVolumeTracings: ServerVolumeTracing[], ): Promise { const promises = serverVolumeTracings .filter((tracing) => tracing.hasEditableMapping) - .map((tracing) => getEditableMappingInfo(tracingStoreUrl, tracing.id)); + .map((tracing) => getEditableMappingInfo(tracingStoreUrl, annotationId, tracing.id)); return Promise.all(promises); } diff --git a/util/src/main/scala/com/scalableminds/util/accesscontext/TokenContext.scala b/util/src/main/scala/com/scalableminds/util/accesscontext/TokenContext.scala new file mode 100644 index 00000000000..2a74b356bf4 --- /dev/null +++ b/util/src/main/scala/com/scalableminds/util/accesscontext/TokenContext.scala @@ -0,0 +1,4 @@ +package com.scalableminds.util.accesscontext + +// to be used in datastore and tracingstore to hand around tokens that were supplied with the request +case class TokenContext(userTokenOpt: Option[String]) diff --git a/util/src/main/scala/com/scalableminds/util/mvc/ExtendedController.scala b/util/src/main/scala/com/scalableminds/util/mvc/ExtendedController.scala index af2a52e2db8..b5f05b3810b 100644 --- a/util/src/main/scala/com/scalableminds/util/mvc/ExtendedController.scala +++ b/util/src/main/scala/com/scalableminds/util/mvc/ExtendedController.scala @@ -1,6 +1,7 @@ package com.scalableminds.util.mvc import com.google.protobuf.CodedInputStream +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.tools.{BoxImplicits, Fox, FoxImplicits} import com.typesafe.scalalogging.LazyLogging import net.liftweb.common._ @@ -235,8 +236,8 @@ trait ValidationHelpers { } trait RequestTokenHelper { - protected def urlOrHeaderToken(token: Option[String], request: Request[Any]): Option[String] = - token.orElse(request.headers.get("X-Auth-Token")) + implicit def tokenContextForRequest(implicit request: Request[Any]): TokenContext = + TokenContext(request.target.getQueryParameter("token").orElse(request.headers.get("X-Auth-Token"))) } trait ExtendedController diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala index bac34609baf..50541f438c9 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala @@ -53,8 +53,7 @@ class BinaryDataController @Inject()( datasetName: String, dataLayerName: String ): Action[List[WebknossosDataRequest]] = Action.async(validateJson[List[WebknossosDataRequest]]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { logTime(slackNotificationService.noticeSlowRequest) { val t = Instant.now for { @@ -96,8 +95,7 @@ class BinaryDataController @Inject()( halfByte: Boolean, mappingName: Option[String] ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -121,8 +119,7 @@ class BinaryDataController @Inject()( datasetName: String, dataLayerName: String ): Action[RawCuboidRequest] = Action.async(validateJson[RawCuboidRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -144,8 +141,7 @@ class BinaryDataController @Inject()( y: Int, z: Int, cubeSize: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -179,8 +175,7 @@ class BinaryDataController @Inject()( intensityMax: Option[Double], color: Option[String], invertColor: Option[Boolean]): Action[RawBuffer] = Action.async(parse.raw) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -227,8 +222,7 @@ class BinaryDataController @Inject()( dataLayerName: String, mappingName: String ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -248,8 +242,7 @@ class BinaryDataController @Inject()( datasetName: String, dataLayerName: String): Action[WebknossosAdHocMeshRequest] = Action.async(validateJson[WebknossosAdHocMeshRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -290,8 +283,7 @@ class BinaryDataController @Inject()( datasetName: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -309,8 +301,7 @@ class BinaryDataController @Inject()( datasetName: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala index f2f4d5921c0..062adc19c8b 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala @@ -28,8 +28,7 @@ class DSMeshController @Inject()( datasetName: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { meshFiles <- meshFileService.exploreMeshFiles(organizationId, datasetName, dataLayerName) } yield Ok(Json.toJson(meshFiles)) @@ -49,8 +48,7 @@ class DSMeshController @Inject()( targetMappingName: Option[String], editableMappingTracingId: Option[String]): Action[ListMeshChunksRequest] = Action.async(validateJson[ListMeshChunksRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { _ <- Fox.successful(()) mappingNameForMeshFile = meshFileService.mappingNameForMeshFile(organizationId, @@ -65,8 +63,7 @@ class DSMeshController @Inject()( editableMappingTracingId, request.body.segmentId, mappingNameForMeshFile, - omitMissing = false, - urlOrHeaderToken(token, request) + omitMissing = false ) chunkInfos <- meshFileService.listMeshChunksForSegmentsMerged(organizationId, datasetName, @@ -82,8 +79,7 @@ class DSMeshController @Inject()( datasetName: String, dataLayerName: String): Action[MeshChunkDataRequestList] = Action.async(validateJson[MeshChunkDataRequestList]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (data, encoding) <- meshFileService.readMeshChunk(organizationId, datasetName, dataLayerName, request.body) ?~> "mesh.file.loadChunk.failed" } yield { @@ -99,14 +95,9 @@ class DSMeshController @Inject()( datasetName: String, dataLayerName: String): Action[FullMeshRequest] = Action.async(validateJson[FullMeshRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { - data: Array[Byte] <- fullMeshService.loadFor(token: Option[String], - organizationId, - datasetName, - dataLayerName, - request.body) ?~> "mesh.file.loadChunk.failed" + data: Array[Byte] <- fullMeshService.loadFor(organizationId, datasetName, dataLayerName, request.body) ?~> "mesh.file.loadChunk.failed" } yield Ok(data) } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala index 2e05d04252d..ff5befccdc3 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala @@ -69,8 +69,7 @@ class DataSourceController @Inject()( Action.async { implicit request => { accessTokenService.validateAccessForSyncBlock( - UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { // Read directly from file, not from repository to ensure recent changes are seen val dataSource: InboxDataSource = dataSourceService.dataSourceFromDir( @@ -82,7 +81,7 @@ class DataSourceController @Inject()( } def triggerInboxCheckBlocking(token: Option[String]): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.administrateDataSources, urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.administrateDataSources) { for { _ <- dataSourceService.checkInbox(verbose = true) } yield Ok @@ -91,12 +90,11 @@ class DataSourceController @Inject()( def reserveUpload(token: Option[String]): Action[ReserveUploadInformation] = Action.async(validateJson[ReserveUploadInformation]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(request.body.organization), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(request.body.organization)) { for { isKnownUpload <- uploadService.isKnownUpload(request.body.uploadId) _ <- if (!isKnownUpload) { - (remoteWebknossosClient.reserveDataSourceUpload(request.body, urlOrHeaderToken(token, request)) ?~> "dataset.upload.validation.failed") + (remoteWebknossosClient.reserveDataSourceUpload(request.body) ?~> "dataset.upload.validation.failed") .flatMap(_ => uploadService.reserveUpload(request.body)) } else Fox.successful(()) } yield Ok @@ -105,11 +103,9 @@ class DataSourceController @Inject()( def getUnfinishedUploads(token: Option[String], organizationName: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(organizationName), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(organizationName)) { for { - unfinishedUploads <- remoteWebknossosClient.getUnfinishedUploadsForUser(urlOrHeaderToken(token, request), - organizationName) + unfinishedUploads <- remoteWebknossosClient.getUnfinishedUploadsForUser(organizationName) unfinishedUploadsWithUploadIds <- uploadService.addUploadIdsToUnfinishedUploads(unfinishedUploads) } yield Ok(Json.toJson(unfinishedUploadsWithUploadIds)) } @@ -119,8 +115,7 @@ class DataSourceController @Inject()( // and it can be put in a webknossos folder where they have access def reserveManualUpload(token: Option[String]): Action[ReserveManualUploadInformation] = Action.async(validateJson[ReserveManualUploadInformation]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(request.body.organization), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(request.body.organization)) { for { _ <- remoteWebknossosClient.reserveDataSourceUpload( ReserveUploadInformation( @@ -132,8 +127,7 @@ class DataSourceController @Inject()( None, request.body.initialTeamIds, request.body.folderId - ), - urlOrHeaderToken(token, request) + ) ) ?~> "dataset.upload.validation.failed" } yield Ok } @@ -172,8 +166,7 @@ class DataSourceController @Inject()( for { dataSourceId <- uploadService.getDataSourceIdByUploadId( uploadService.extractDatasetUploadId(uploadFileId)) ?~> "dataset.upload.validation.failed" - result <- accessTokenService.validateAccess(UserAccessRequest.writeDataSource(dataSourceId), - urlOrHeaderToken(token, request)) { + result <- accessTokenService.validateAccess(UserAccessRequest.writeDataSource(dataSourceId)) { for { isKnownUpload <- uploadService.isKnownUploadByFileId(uploadFileId) _ <- bool2Fox(isKnownUpload) ?~> "dataset.upload.validation.failed" @@ -195,8 +188,7 @@ class DataSourceController @Inject()( for { dataSourceId <- uploadService.getDataSourceIdByUploadId( uploadService.extractDatasetUploadId(resumableIdentifier)) ?~> "dataset.upload.validation.failed" - result <- accessTokenService.validateAccess(UserAccessRequest.writeDataSource(dataSourceId), - urlOrHeaderToken(token, request)) { + result <- accessTokenService.validateAccess(UserAccessRequest.writeDataSource(dataSourceId)) { for { isKnownUpload <- uploadService.isKnownUploadByFileId(resumableIdentifier) _ <- bool2Fox(isKnownUpload) ?~> "dataset.upload.validation.failed" @@ -212,16 +204,13 @@ class DataSourceController @Inject()( for { dataSourceId <- uploadService .getDataSourceIdByUploadId(request.body.uploadId) ?~> "dataset.upload.validation.failed" - result <- accessTokenService.validateAccess(UserAccessRequest.writeDataSource(dataSourceId), - urlOrHeaderToken(token, request)) { + result <- accessTokenService.validateAccess(UserAccessRequest.writeDataSource(dataSourceId)) { for { (dataSourceId, datasetSizeBytes) <- uploadService.finishUpload(request.body) ?~> "finishUpload.failed" - _ <- remoteWebknossosClient.reportUpload( - dataSourceId, - datasetSizeBytes, - request.body.needsConversion.getOrElse(false), - viaAddRoute = false, - userToken = urlOrHeaderToken(token, request)) ?~> "reportUpload.failed" + _ <- remoteWebknossosClient.reportUpload(dataSourceId, + datasetSizeBytes, + request.body.needsConversion.getOrElse(false), + viaAddRoute = false) ?~> "reportUpload.failed" } yield Ok } } yield result @@ -235,8 +224,7 @@ class DataSourceController @Inject()( case true => uploadService.getDataSourceIdByUploadId(request.body.uploadId) } dataSourceIdFox.flatMap { dataSourceId => - accessTokenService.validateAccess(UserAccessRequest.deleteDataSource(dataSourceId), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.deleteDataSource(dataSourceId)) { for { _ <- remoteWebknossosClient.deleteDataSource(dataSourceId) ?~> "dataset.delete.webknossos.failed" _ <- uploadService.cancelUpload(request.body) ?~> "Could not cancel the upload." @@ -252,8 +240,7 @@ class DataSourceController @Inject()( dataLayerName: String ): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessForSyncBlock( - UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { addNoCacheHeaderFallback( Ok(Json.toJson(dataSourceService.exploreMappings(organizationId, datasetName, dataLayerName)))) } @@ -265,8 +252,7 @@ class DataSourceController @Inject()( datasetName: String, dataLayerName: String ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox agglomerateList = agglomerateService.exploreAgglomerates(organizationId, datasetName, dataLayerName) @@ -282,8 +268,7 @@ class DataSourceController @Inject()( mappingName: String, agglomerateId: Long ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox skeleton <- agglomerateService.generateSkeleton(organizationId, @@ -303,8 +288,7 @@ class DataSourceController @Inject()( mappingName: String, agglomerateId: Long ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox agglomerateGraph <- agglomerateService.generateAgglomerateGraph( @@ -322,8 +306,7 @@ class DataSourceController @Inject()( mappingName: String, segmentId: Long ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox position <- agglomerateService.positionForSegmentId( @@ -340,8 +323,7 @@ class DataSourceController @Inject()( dataLayerName: String, mappingName: String ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox largestAgglomerateId: Long <- agglomerateService @@ -365,8 +347,7 @@ class DataSourceController @Inject()( dataLayerName: String, mappingName: String ): Action[ListOfLong] = Action.async(validateProto[ListOfLong]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox agglomerateIds: Seq[Long] <- agglomerateService @@ -391,8 +372,7 @@ class DataSourceController @Inject()( dataLayerName: String, mappingName: String ): Action[ListOfLong] = Action.async(validateProto[ListOfLong]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox agglomerateIds: Array[Long] <- agglomerateService @@ -411,8 +391,7 @@ class DataSourceController @Inject()( def update(token: Option[String], organizationId: String, datasetName: String): Action[DataSource] = Action.async(validateJson[DataSource]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.writeDataSource(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.writeDataSource(DataSourceId(datasetName, organizationId))) { for { _ <- Fox.successful(()) dataSource <- dataSourceRepository.find(DataSourceId(datasetName, organizationId)).toFox ?~> Messages( @@ -428,7 +407,7 @@ class DataSourceController @Inject()( datasetName: String, folderId: Option[String]): Action[DataSource] = Action.async(validateJson[DataSource]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.administrateDataSources, urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.administrateDataSources) { for { _ <- bool2Fox(dataSourceRepository.find(DataSourceId(datasetName, organizationId)).isEmpty) ?~> Messages( "dataSource.alreadyPresent") @@ -442,24 +421,21 @@ class DataSourceController @Inject()( layersToLink = None, initialTeams = List.empty, folderId = folderId, - ), - urlOrHeaderToken(token, request) + ) ) ?~> "dataset.upload.validation.failed" _ <- dataSourceService.updateDataSource(request.body.copy(id = DataSourceId(datasetName, organizationId)), expectExisting = false) - _ <- remoteWebknossosClient.reportUpload( - DataSourceId(datasetName, organizationId), - 0L, - needsConversion = false, - viaAddRoute = true, - userToken = urlOrHeaderToken(token, request)) ?~> "reportUpload.failed" + _ <- remoteWebknossosClient.reportUpload(DataSourceId(datasetName, organizationId), + 0L, + needsConversion = false, + viaAddRoute = true) ?~> "reportUpload.failed" } yield Ok } } def createOrganizationDirectory(token: Option[String], organizationId: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessForSyncBlock(UserAccessRequest.administrateDataSources(organizationId), token) { + accessTokenService.validateAccessForSyncBlock(UserAccessRequest.administrateDataSources(organizationId)) { val newOrganizationDirectory = new File(f"${dataSourceService.dataBaseDir}/$organizationId") newOrganizationDirectory.mkdirs() if (newOrganizationDirectory.isDirectory) @@ -474,8 +450,7 @@ class DataSourceController @Inject()( datasetName: Option[String] = None): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(organizationId), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(organizationId)) { for { before <- Fox.successful(System.currentTimeMillis()) usedStorageInBytes: List[DirectoryStorageReport] <- storageUsageService.measureStorage(organizationId, @@ -495,8 +470,7 @@ class DataSourceController @Inject()( datasetName: String, layerName: Option[String] = None): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(organizationId), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(organizationId)) { val (closedAgglomerateFileHandleCount, clearedBucketProviderCount, removedChunksCount) = binaryDataServiceHolder.binaryDataService.clearCache(organizationId, datasetName, layerName) val reloadedDataSource = dataSourceService.dataSourceFromDir( @@ -519,8 +493,7 @@ class DataSourceController @Inject()( def deleteOnDisk(token: Option[String], organizationId: String, datasetName: String): Action[AnyContent] = Action.async { implicit request => val dataSourceId = DataSourceId(datasetName, organizationId) - accessTokenService.validateAccess(UserAccessRequest.deleteDataSource(dataSourceId), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.deleteDataSource(dataSourceId)) { for { _ <- binaryDataServiceHolder.binaryDataService.deleteOnDisk( organizationId, @@ -533,15 +506,12 @@ class DataSourceController @Inject()( def compose(token: Option[String]): Action[ComposeRequest] = Action.async(validateJson[ComposeRequest]) { implicit request => - val userToken = urlOrHeaderToken(token, request) - accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(request.body.organizationId), token) { + accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(request.body.organizationId)) { for { - _ <- Fox.serialCombined(request.body.layers.map(_.datasetId).toList)( - id => - accessTokenService.assertUserAccess( - UserAccessRequest.readDataSources(DataSourceId(id.name, id.owningOrganization)), - userToken)) - dataSource <- composeService.composeDataset(request.body, userToken) + _ <- Fox.serialCombined(request.body.layers.map(_.datasetId).toList)(id => + accessTokenService.assertUserAccess( + UserAccessRequest.readDataSources(DataSourceId(id.name, id.owningOrganization)))) + dataSource <- composeService.composeDataset(request.body) _ <- dataSourceRepository.updateDataSource(dataSource) } yield Ok } @@ -552,8 +522,7 @@ class DataSourceController @Inject()( datasetName: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { val connectomeFileNames = connectomeFileService.exploreConnectomeFiles(organizationId, datasetName, dataLayerName) for { @@ -574,8 +543,7 @@ class DataSourceController @Inject()( datasetName: String, dataLayerName: String): Action[ByAgglomerateIdsRequest] = Action.async(validateJson[ByAgglomerateIdsRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { meshFilePath <- Fox.successful( connectomeFileService @@ -591,8 +559,7 @@ class DataSourceController @Inject()( dataLayerName: String, direction: String): Action[BySynapseIdsRequest] = Action.async(validateJson[BySynapseIdsRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { meshFilePath <- Fox.successful( connectomeFileService @@ -609,8 +576,7 @@ class DataSourceController @Inject()( datasetName: String, dataLayerName: String): Action[BySynapseIdsRequest] = Action.async(validateJson[BySynapseIdsRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { meshFilePath <- Fox.successful( connectomeFileService @@ -625,8 +591,7 @@ class DataSourceController @Inject()( datasetName: String, dataLayerName: String): Action[BySynapseIdsRequest] = Action.async(validateJson[BySynapseIdsRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { meshFilePath <- Fox.successful( connectomeFileService @@ -641,8 +606,7 @@ class DataSourceController @Inject()( dataSetName: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(dataSetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(dataSetName, organizationId))) { val segmentIndexFileOpt = segmentIndexFileService.getSegmentIndexFile(organizationId, dataSetName, dataLayerName).toOption Future.successful(Ok(Json.toJson(segmentIndexFileOpt.isDefined))) @@ -659,8 +623,7 @@ class DataSourceController @Inject()( dataLayerName: String, segmentId: String): Action[GetSegmentIndexParameters] = Action.async(validateJson[GetSegmentIndexParameters]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { segmentIds <- segmentIdsForAgglomerateIdIfNeeded( organizationId, @@ -670,8 +633,7 @@ class DataSourceController @Inject()( request.body.editableMappingTracingId, segmentId.toLong, mappingNameForMeshFile = None, - omitMissing = false, - urlOrHeaderToken(token, request) + omitMissing = false ) fileMag <- segmentIndexFileService.readFileMag(organizationId, datasetName, dataLayerName) topLeftsNested: Seq[Array[Vec3Int]] <- Fox.serialCombined(segmentIds)(sId => @@ -698,8 +660,7 @@ class DataSourceController @Inject()( datasetName: String, dataLayerName: String): Action[GetMultipleSegmentIndexParameters] = Action.async(validateJson[GetMultipleSegmentIndexParameters]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { segmentIdsAndBucketPositions <- Fox.serialCombined(request.body.segmentIds) { segmentOrAgglomerateId => for { @@ -712,7 +673,6 @@ class DataSourceController @Inject()( segmentOrAgglomerateId, mappingNameForMeshFile = None, omitMissing = true, // assume agglomerate ids not present in the mapping belong to user-brushed segments - urlOrHeaderToken(token, request) ) fileMag <- segmentIndexFileService.readFileMag(organizationId, datasetName, dataLayerName) topLeftsNested: Seq[Array[Vec3Int]] <- Fox.serialCombined(segmentIds)(sId => @@ -732,8 +692,7 @@ class DataSourceController @Inject()( datasetName: String, dataLayerName: String): Action[SegmentStatisticsParameters] = Action.async(validateJson[SegmentStatisticsParameters]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { _ <- segmentIndexFileService.assertSegmentIndexFileExists(organizationId, datasetName, dataLayerName) volumes <- Fox.serialCombined(request.body.segmentIds) { segmentId => @@ -755,8 +714,7 @@ class DataSourceController @Inject()( datasetName: String, dataLayerName: String): Action[SegmentStatisticsParameters] = Action.async(validateJson[SegmentStatisticsParameters]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { _ <- segmentIndexFileService.assertSegmentIndexFileExists(organizationId, datasetName, dataLayerName) boxes <- Fox.serialCombined(request.body.segmentIds) { segmentId => @@ -774,7 +732,7 @@ class DataSourceController @Inject()( // Called directly by wk side def exploreRemoteDataset(token: Option[String]): Action[ExploreRemoteDatasetRequest] = Action.async(validateJson[ExploreRemoteDatasetRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(request.body.organizationId), token) { + accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(request.body.organizationId)) { val reportMutable = ListBuffer[String]() val hasLocalFilesystemRequest = request.body.layerParameters.exists(param => new URI(param.remoteUri).getScheme == DataVaultService.schemeFile) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ExportsController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ExportsController.scala index f4777fdb4b9..009ac58d0f5 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ExportsController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ExportsController.scala @@ -36,7 +36,7 @@ class ExportsController @Inject()(webknossosClient: DSRemoteWebknossosClient, override def allowRemoteOrigin: Boolean = true def download(token: Option[String], jobId: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.downloadJobExport(jobId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.downloadJobExport(jobId)) { for { exportProperties <- webknossosClient.getJobExportProperties(jobId) fullPath = exportProperties.fullPathIn(dataBaseDir) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala index 2c94c3386e6..1f86aef3d08 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala @@ -1,6 +1,7 @@ package com.scalableminds.webknossos.datastore.controllers import com.google.inject.Inject +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.geometry.Vec3Int import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.dataformats.MagLocator @@ -55,8 +56,7 @@ class ZarrStreamingController @Inject()( datasetName: String, dataLayerName: String = "", ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -73,8 +73,7 @@ class ZarrStreamingController @Inject()( datasetName: String, dataLayerName: String = "", ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -94,12 +93,11 @@ class ZarrStreamingController @Inject()( dataLayerName: String = ""): Action[AnyContent] = Action.async { implicit request => ifIsAnnotationLayerOrElse( - token, accessToken, dataLayerName, - ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => { + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantTokenContext) => { remoteTracingstoreClient - .getOmeNgffHeader(annotationLayer.tracingId, annotationSource.tracingStoreUrl, relevantToken) + .getOmeNgffHeader(annotationLayer.tracingId, annotationSource.tracingStoreUrl)(relevantTokenContext) .map(ngffMetadata => Ok(Json.toJson(ngffMetadata))) }, orElse = annotationSource => @@ -120,12 +118,12 @@ class ZarrStreamingController @Inject()( dataLayerName: String = ""): Action[AnyContent] = Action.async { implicit request => ifIsAnnotationLayerOrElse( - token, accessToken, dataLayerName, - ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => { + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantTokenContext) => { remoteTracingstoreClient - .getZarrJsonGroupHeaderWithNgff(annotationLayer.tracingId, annotationSource.tracingStoreUrl, relevantToken) + .getZarrJsonGroupHeaderWithNgff(annotationLayer.tracingId, annotationSource.tracingStoreUrl)( + relevantTokenContext) .map(header => Ok(Json.toJson(header))) }, orElse = annotationSource => @@ -153,8 +151,7 @@ class ZarrStreamingController @Inject()( datasetName: String, zarrVersion: Int, ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { dataSource <- dataSourceRepository.findUsable(DataSourceId(datasetName, organizationId)).toFox ~> NOT_FOUND dataLayers = dataSource.dataLayers @@ -206,9 +203,9 @@ class ZarrStreamingController @Inject()( zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) - else urlOrHeaderToken(token, request) + annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken) ~> NOT_FOUND + relevantTokenContext = if (annotationSource.accessViaPrivateLink) TokenContext(Some(accessToken)) + else tokenContextForRequest volumeAnnotationLayers = annotationSource.annotationLayers.filter(_.typ == AnnotationLayerType.Volume) dataSource <- dataSourceRepository .findUsable(DataSourceId(annotationSource.datasetName, annotationSource.organizationId)) @@ -221,8 +218,7 @@ class ZarrStreamingController @Inject()( remoteTracingstoreClient.getVolumeLayerAsZarrLayer(l.tracingId, Some(l.name), annotationSource.tracingStoreUrl, - relevantToken, - zarrVersion)) + zarrVersion)(relevantTokenContext)) allLayer = dataSourceLayers ++ annotationLayers zarrSource = GenericDataSource[DataLayer](dataSource.id, allLayer, dataSource.scale) } yield Ok(Json.toJson(zarrSource)) @@ -236,8 +232,7 @@ class ZarrStreamingController @Inject()( mag: String, coordinates: String, ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { rawZarrCube(organizationId, datasetName, dataLayerName, mag, coordinates) } } @@ -249,16 +244,12 @@ class ZarrStreamingController @Inject()( coordinates: String): Action[AnyContent] = Action.async { implicit request => ifIsAnnotationLayerOrElse( - token, accessToken, dataLayerName, - ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantTokenContext) => remoteTracingstoreClient - .getRawZarrCube(annotationLayer.tracingId, - mag, - coordinates, - annotationSource.tracingStoreUrl, - relevantToken) + .getRawZarrCube(annotationLayer.tracingId, mag, coordinates, annotationSource.tracingStoreUrl)( + relevantTokenContext) .map(Ok(_)), orElse = annotationSource => rawZarrCube(annotationSource.organizationId, annotationSource.datasetName, dataLayerName, mag, coordinates) @@ -307,8 +298,7 @@ class ZarrStreamingController @Inject()( dataLayerName: String, mag: String, ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { zArray(organizationId, datasetName, dataLayerName, mag) } } @@ -329,8 +319,7 @@ class ZarrStreamingController @Inject()( dataLayerName: String, mag: String, ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { zarrJsonForMag(organizationId, datasetName, dataLayerName, mag) } } @@ -350,12 +339,11 @@ class ZarrStreamingController @Inject()( dataLayerName: String, mag: String): Action[AnyContent] = Action.async { implicit request => ifIsAnnotationLayerOrElse( - token, accessToken, dataLayerName, - ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantTokenContext) => remoteTracingstoreClient - .getZArray(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl, relevantToken) + .getZArray(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl)(relevantTokenContext) .map(z => Ok(Json.toJson(z))), orElse = annotationSource => zArray(annotationSource.organizationId, annotationSource.datasetName, dataLayerName, mag) @@ -367,12 +355,11 @@ class ZarrStreamingController @Inject()( dataLayerName: String, mag: String): Action[AnyContent] = Action.async { implicit request => ifIsAnnotationLayerOrElse( - token, accessToken, dataLayerName, - ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantTokenContext) => remoteTracingstoreClient - .getZarrJson(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl, relevantToken) + .getZarrJson(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl)(relevantTokenContext) .map(z => Ok(Json.toJson(z))), orElse = annotationSource => zarrJsonForMag(annotationSource.organizationId, annotationSource.datasetName, dataLayerName, mag) @@ -380,18 +367,17 @@ class ZarrStreamingController @Inject()( } private def ifIsAnnotationLayerOrElse( - token: Option[String], accessToken: String, dataLayerName: String, - ifIsAnnotationLayer: (AnnotationLayer, AnnotationSource, Option[String]) => Fox[Result], + ifIsAnnotationLayer: (AnnotationLayer, AnnotationSource, TokenContext) => Fox[Result], orElse: AnnotationSource => Fox[Result])(implicit request: Request[Any]): Fox[Result] = for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) ~> NOT_FOUND - relevantToken = if (annotationSource.accessViaPrivateLink) Some(accessToken) - else urlOrHeaderToken(token, request) + annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken) ~> NOT_FOUND + relevantTokenContext = if (annotationSource.accessViaPrivateLink) TokenContext(Some(accessToken)) + else tokenContextForRequest layer = annotationSource.getAnnotationLayer(dataLayerName) result <- layer match { - case Some(annotationLayer) => ifIsAnnotationLayer(annotationLayer, annotationSource, relevantToken) + case Some(annotationLayer) => ifIsAnnotationLayer(annotationLayer, annotationSource, relevantTokenContext) case None => orElse(annotationSource) } } yield result @@ -403,8 +389,7 @@ class ZarrStreamingController @Inject()( mag: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { dataLayerMagFolderContents(organizationId, datasetName, dataLayerName, mag, zarrVersion) } } @@ -435,16 +420,14 @@ class ZarrStreamingController @Inject()( zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => ifIsAnnotationLayerOrElse( - token, accessToken, dataLayerName, - ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantTokenContext) => remoteTracingstoreClient .getDataLayerMagFolderContents(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl, - relevantToken, - zarrVersion) + zarrVersion)(relevantTokenContext) .map( layers => Ok( @@ -467,8 +450,7 @@ class ZarrStreamingController @Inject()( datasetName: String, dataLayerName: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { dataLayerFolderContents(organizationId, datasetName, dataLayerName, zarrVersion) } } @@ -498,15 +480,12 @@ class ZarrStreamingController @Inject()( zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => ifIsAnnotationLayerOrElse( - token, accessToken, dataLayerName, - ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantTokenContext) => remoteTracingstoreClient - .getDataLayerFolderContents(annotationLayer.tracingId, - annotationSource.tracingStoreUrl, - relevantToken, - zarrVersion) + .getDataLayerFolderContents(annotationLayer.tracingId, annotationSource.tracingStoreUrl, zarrVersion)( + relevantTokenContext) .map( layers => Ok( @@ -528,8 +507,7 @@ class ZarrStreamingController @Inject()( datasetName: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { dataSource <- dataSourceRepository.findUsable(DataSourceId(datasetName, organizationId)).toFox ?~> Messages( "dataSource.notFound") ~> NOT_FOUND @@ -550,7 +528,7 @@ class ZarrStreamingController @Inject()( zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => for { - annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken, urlOrHeaderToken(token, request)) + annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken) dataSource <- dataSourceRepository .findUsable(DataSourceId(annotationSource.datasetName, annotationSource.organizationId)) .toFox ?~> Messages("dataSource.notFound") ~> NOT_FOUND @@ -577,8 +555,7 @@ class ZarrStreamingController @Inject()( datasetName: String, dataLayerName: String = ""): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessForSyncBlock( - UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId)), - urlOrHeaderToken(token, request)) { + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { Ok(zGroupJson) } } @@ -588,12 +565,11 @@ class ZarrStreamingController @Inject()( def zGroupPrivateLink(token: Option[String], accessToken: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => ifIsAnnotationLayerOrElse( - token, accessToken, dataLayerName, - ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantToken) => + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantTokenContext) => remoteTracingstoreClient - .getZGroup(annotationLayer.tracingId, annotationSource.tracingStoreUrl, relevantToken) + .getZGroup(annotationLayer.tracingId, annotationSource.tracingStoreUrl)(relevantTokenContext) .map(Ok(_)), orElse = _ => Fox.successful(Ok(zGroupJson)) ) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala index 1d99f8aab03..d26675b7e6d 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala @@ -1,5 +1,6 @@ package com.scalableminds.webknossos.datastore.rpc +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.mvc.MimeTypes import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.typesafe.scalalogging.LazyLogging @@ -26,6 +27,9 @@ class RPCRequest(val id: Int, val url: String, wsClient: WSClient)(implicit ec: this } + def withTokenFromContext(implicit tc: TokenContext): RPCRequest = + addQueryStringOptional("token", tc.userTokenOpt) + def addHttpHeaders(hdrs: (String, String)*): RPCRequest = { request = request.addHttpHeaders(hdrs: _*) this diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AccessTokenService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AccessTokenService.scala index 63db94987be..52b56268077 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AccessTokenService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AccessTokenService.scala @@ -1,6 +1,7 @@ package com.scalableminds.webknossos.datastore.services import com.google.inject.Inject +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.enumeration.ExtendedEnumeration import com.scalableminds.util.tools.Fox @@ -66,28 +67,27 @@ trait AccessTokenService { private lazy val accessAnswersCache: AlfuCache[(UserAccessRequest, Option[String]), UserAccessAnswer] = AlfuCache(timeToLive = AccessExpiration, timeToIdle = AccessExpiration) - def validateAccessForSyncBlock(accessRequest: UserAccessRequest, token: Option[String])(block: => Result)( - implicit ec: ExecutionContext): Fox[Result] = - validateAccess(accessRequest, token) { + def validateAccessForSyncBlock(accessRequest: UserAccessRequest)(block: => Result)(implicit ec: ExecutionContext, + tc: TokenContext): Fox[Result] = + validateAccess(accessRequest) { Future.successful(block) } - def validateAccess(accessRequest: UserAccessRequest, token: Option[String])(block: => Future[Result])( - implicit ec: ExecutionContext): Fox[Result] = + def validateAccess(accessRequest: UserAccessRequest)(block: => Future[Result])(implicit ec: ExecutionContext, + tc: TokenContext): Fox[Result] = for { - userAccessAnswer <- hasUserAccess(accessRequest, token) ?~> "Failed to check data access, token may be expired, consider reloading." + userAccessAnswer <- hasUserAccess(accessRequest) ?~> "Failed to check data access, token may be expired, consider reloading." result <- executeBlockOnPositiveAnswer(userAccessAnswer, block) } yield result - private def hasUserAccess(accessRequest: UserAccessRequest, token: Option[String])( - implicit ec: ExecutionContext): Fox[UserAccessAnswer] = - accessAnswersCache.getOrLoad((accessRequest, token), - _ => remoteWebknossosClient.requestUserAccess(token, accessRequest)) + private def hasUserAccess(accessRequest: UserAccessRequest)(implicit ec: ExecutionContext, + tc: TokenContext): Fox[UserAccessAnswer] = + accessAnswersCache.getOrLoad((accessRequest, tc.userTokenOpt), + _ => remoteWebknossosClient.requestUserAccess(accessRequest)) - def assertUserAccess(accessRequest: UserAccessRequest, token: Option[String])( - implicit ec: ExecutionContext): Fox[Unit] = + def assertUserAccess(accessRequest: UserAccessRequest)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Unit] = for { - userAccessAnswer <- hasUserAccess(accessRequest, token) ?~> "Failed to check data access, token may be expired, consider reloading." + userAccessAnswer <- hasUserAccess(accessRequest) ?~> "Failed to check data access, token may be expired, consider reloading." _ <- Fox.bool2Fox(userAccessAnswer.granted) ?~> userAccessAnswer.msg.getOrElse("Access forbidden.") } yield () diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSFullMeshService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSFullMeshService.scala index 24c14630a91..c0af56127ed 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSFullMeshService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSFullMeshService.scala @@ -1,6 +1,7 @@ package com.scalableminds.webknossos.datastore.services import com.google.inject.Inject +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.geometry.{Vec3Double, Vec3Int} import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox @@ -49,14 +50,13 @@ class DSFullMeshService @Inject()(dataSourceRepository: DataSourceRepository, (binaryDataService, mappingService, config.Datastore.AdHocMesh.timeout, config.Datastore.AdHocMesh.actorPoolSize) val adHocMeshService: AdHocMeshService = adHocMeshServiceHolder.dataStoreAdHocMeshService - def loadFor(token: Option[String], - organizationId: String, - datasetName: String, - dataLayerName: String, - fullMeshRequest: FullMeshRequest)(implicit ec: ExecutionContext, m: MessagesProvider): Fox[Array[Byte]] = + def loadFor(organizationId: String, datasetName: String, dataLayerName: String, fullMeshRequest: FullMeshRequest)( + implicit ec: ExecutionContext, + m: MessagesProvider, + tc: TokenContext): Fox[Array[Byte]] = fullMeshRequest.meshFileName match { case Some(_) => - loadFullMeshFromMeshfile(token, organizationId, datasetName, dataLayerName, fullMeshRequest) + loadFullMeshFromMeshfile(organizationId, datasetName, dataLayerName, fullMeshRequest) case None => loadFullMeshFromAdHoc(organizationId, datasetName, dataLayerName, fullMeshRequest) } @@ -113,12 +113,12 @@ class DSFullMeshService @Inject()(dataSourceRepository: DataSourceRepository, } yield allVertices } - private def loadFullMeshFromMeshfile( - token: Option[String], - organizationId: String, - datasetName: String, - layerName: String, - fullMeshRequest: FullMeshRequest)(implicit ec: ExecutionContext, m: MessagesProvider): Fox[Array[Byte]] = + private def loadFullMeshFromMeshfile(organizationId: String, + datasetName: String, + layerName: String, + fullMeshRequest: FullMeshRequest)(implicit ec: ExecutionContext, + m: MessagesProvider, + tc: TokenContext): Fox[Array[Byte]] = for { meshFileName <- fullMeshRequest.meshFileName.toFox ?~> "meshFileName.needed" before = Instant.now @@ -134,8 +134,7 @@ class DSFullMeshService @Inject()(dataSourceRepository: DataSourceRepository, fullMeshRequest.editableMappingTracingId, fullMeshRequest.segmentId, mappingNameForMeshFile, - omitMissing = false, - token + omitMissing = false ) chunkInfos: WebknossosSegmentInfo <- meshFileService.listMeshChunksForSegmentsMerged(organizationId, datasetName, diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala index 2924c0687e4..5bd69d4d7c9 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala @@ -1,8 +1,8 @@ package com.scalableminds.webknossos.datastore.services import com.google.inject.Inject +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.tools.{Fox, FoxImplicits} -import com.scalableminds.webknossos.datastore.DataStoreConfig import com.scalableminds.webknossos.datastore.dataformats.layers.ZarrSegmentationLayer import com.scalableminds.webknossos.datastore.datareaders.zarr.{NgffMetadata, ZarrHeader} import com.scalableminds.webknossos.datastore.datareaders.zarr3.{Zarr3ArrayHeader, Zarr3GroupHeader} @@ -21,88 +21,63 @@ object EditableMappingSegmentListResult { class DSRemoteTracingstoreClient @Inject()( rpc: RPC, - config: DataStoreConfig, val lifecycle: ApplicationLifecycle, ) extends LazyLogging with FoxImplicits { + private def getZarrVersionDependantSubPath = (zarrVersion: Int) => if (zarrVersion == 2) "zarr" else "zarr3_experimental" - def getZArray(tracingId: String, mag: String, tracingStoreUri: String, token: Option[String]): Fox[ZarrHeader] = - rpc(s"$tracingStoreUri/tracings/volume/zarr/$tracingId/$mag/.zarray") - .addQueryStringOptional("token", token) + def getZArray(tracingId: String, mag: String, tracingStoreUri: String)(implicit tc: TokenContext): Fox[ZarrHeader] = + rpc(s"$tracingStoreUri/tracings/volume/zarr/$tracingId/$mag/.zarray").withTokenFromContext .getWithJsonResponse[ZarrHeader] - def getZarrJson(tracingId: String, - mag: String, - tracingStoreUri: String, - token: Option[String]): Fox[Zarr3ArrayHeader] = - rpc(s"$tracingStoreUri/tracings/volume/zarr3_experimental/$tracingId/$mag/zarr.json") - .addQueryStringOptional("token", token) + def getZarrJson(tracingId: String, mag: String, tracingStoreUri: String)( + implicit tc: TokenContext): Fox[Zarr3ArrayHeader] = + rpc(s"$tracingStoreUri/tracings/volume/zarr3_experimental/$tracingId/$mag/zarr.json").withTokenFromContext .getWithJsonResponse[Zarr3ArrayHeader] def getVolumeLayerAsZarrLayer(tracingId: String, tracingName: Option[String], tracingStoreUri: String, - token: Option[String], - zarrVersion: Int): Fox[ZarrSegmentationLayer] = { + zarrVersion: Int)(implicit tc: TokenContext): Fox[ZarrSegmentationLayer] = { val zarrVersionDependantSubPath = getZarrVersionDependantSubPath(zarrVersion) - rpc(s"$tracingStoreUri/tracings/volume/$zarrVersionDependantSubPath/$tracingId/zarrSource") - .addQueryStringOptional("token", token) + rpc(s"$tracingStoreUri/tracings/volume/$zarrVersionDependantSubPath/$tracingId/zarrSource").withTokenFromContext .addQueryStringOptional("tracingName", tracingName) .getWithJsonResponse[ZarrSegmentationLayer] } - def getOmeNgffHeader(tracingId: String, tracingStoreUri: String, token: Option[String]): Fox[NgffMetadata] = - rpc(s"$tracingStoreUri/tracings/volume/zarr/$tracingId/.zattrs") - .addQueryStringOptional("token", token) + def getOmeNgffHeader(tracingId: String, tracingStoreUri: String)(implicit tc: TokenContext): Fox[NgffMetadata] = + rpc(s"$tracingStoreUri/tracings/volume/zarr/$tracingId/.zattrs").withTokenFromContext .getWithJsonResponse[NgffMetadata] - def getZarrJsonGroupHeaderWithNgff(tracingId: String, - tracingStoreUri: String, - token: Option[String]): Fox[Zarr3GroupHeader] = - rpc(s"$tracingStoreUri/tracings/volume/zarr3_experimental/$tracingId/zarr.json") - .addQueryStringOptional("token", token) + def getZarrJsonGroupHeaderWithNgff(tracingId: String, tracingStoreUri: String)( + implicit tc: TokenContext): Fox[Zarr3GroupHeader] = + rpc(s"$tracingStoreUri/tracings/volume/zarr3_experimental/$tracingId/zarr.json").withTokenFromContext .getWithJsonResponse[Zarr3GroupHeader] - def getRawZarrCube(tracingId: String, - mag: String, - cxyz: String, - tracingStoreUri: String, - token: Option[String]): Fox[Array[Byte]] = - rpc(s"$tracingStoreUri/tracings/volume/zarr/$tracingId/$mag/$cxyz").silent - .addQueryStringOptional("token", token) - .getWithBytesResponse + def getRawZarrCube(tracingId: String, mag: String, cxyz: String, tracingStoreUri: String)( + implicit tc: TokenContext): Fox[Array[Byte]] = + rpc(s"$tracingStoreUri/tracings/volume/zarr/$tracingId/$mag/$cxyz").silent.withTokenFromContext.getWithBytesResponse - def getDataLayerMagFolderContents(tracingId: String, - mag: String, - tracingStoreUri: String, - token: Option[String], - zarrVersion: Int): Fox[List[String]] = - rpc(s"$tracingStoreUri/tracings/volume/${getZarrVersionDependantSubPath(zarrVersion)}/json/$tracingId/$mag") - .addQueryStringOptional("token", token) + def getDataLayerMagFolderContents(tracingId: String, mag: String, tracingStoreUri: String, zarrVersion: Int)( + implicit tc: TokenContext): Fox[List[String]] = + rpc(s"$tracingStoreUri/tracings/volume/${getZarrVersionDependantSubPath(zarrVersion)}/json/$tracingId/$mag").withTokenFromContext .getWithJsonResponse[List[String]] - def getDataLayerFolderContents(tracingId: String, - tracingStoreUri: String, - token: Option[String], - zarrVersion: Int): Fox[List[String]] = - rpc(s"$tracingStoreUri/tracings/volume/${getZarrVersionDependantSubPath(zarrVersion)}/json/$tracingId") - .addQueryStringOptional("token", token) + def getDataLayerFolderContents(tracingId: String, tracingStoreUri: String, zarrVersion: Int)( + implicit tc: TokenContext): Fox[List[String]] = + rpc(s"$tracingStoreUri/tracings/volume/${getZarrVersionDependantSubPath(zarrVersion)}/json/$tracingId").withTokenFromContext .getWithJsonResponse[List[String]] - def getZGroup(tracingId: String, tracingStoreUri: String, token: Option[String]): Fox[JsObject] = - rpc(s"$tracingStoreUri/tracings/volume/zarr/$tracingId/.zgroup") - .addQueryStringOptional("token", token) - .getWithJsonResponse[JsObject] + def getZGroup(tracingId: String, tracingStoreUri: String)(implicit tc: TokenContext): Fox[JsObject] = + rpc(s"$tracingStoreUri/tracings/volume/zarr/$tracingId/.zgroup").withTokenFromContext.getWithJsonResponse[JsObject] - def getEditableMappingSegmentIdsForAgglomerate(tracingStoreUri: String, - tracingId: String, - agglomerateId: Long, - token: Option[String]): Fox[EditableMappingSegmentListResult] = + def getEditableMappingSegmentIdsForAgglomerate(tracingStoreUri: String, tracingId: String, agglomerateId: Long)( + implicit tc: TokenContext): Fox[EditableMappingSegmentListResult] = rpc(s"$tracingStoreUri/tracings/mapping/$tracingId/segmentsForAgglomerate") .addQueryString("agglomerateId" -> agglomerateId.toString) - .addQueryStringOptional("token", token) + .withTokenFromContext .silent .getWithJsonResponse[EditableMappingSegmentListResult] } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebknossosClient.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebknossosClient.scala index 0703f638076..b4bcafed613 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebknossosClient.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebknossosClient.scala @@ -3,6 +3,7 @@ package com.scalableminds.webknossos.datastore.services import org.apache.pekko.actor.ActorSystem import com.google.inject.Inject import com.google.inject.name.Named +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.datastore.DataStoreConfig @@ -33,7 +34,7 @@ object TracingStoreInfo { } trait RemoteWebknossosClient { - def requestUserAccess(token: Option[String], accessRequest: UserAccessRequest): Fox[UserAccessAnswer] + def requestUserAccess(accessRequest: UserAccessRequest)(implicit tc: TokenContext): Fox[UserAccessAnswer] } class DSRemoteWebknossosClient @Inject()( @@ -68,21 +69,17 @@ class DSRemoteWebknossosClient @Inject()( .addQueryString("key" -> dataStoreKey) .put(dataSource) - def getUnfinishedUploadsForUser(userTokenOpt: Option[String], organizationName: String): Fox[List[UnfinishedUpload]] = + def getUnfinishedUploadsForUser(organizationName: String)(implicit tc: TokenContext): Fox[List[UnfinishedUpload]] = for { - userToken <- option2Fox(userTokenOpt) ?~> "reserveUpload.noUserToken" unfinishedUploads <- rpc(s"$webknossosUri/api/datastores/$dataStoreName/getUnfinishedUploadsForUser") .addQueryString("key" -> dataStoreKey) - .addQueryString("token" -> userToken) .addQueryString("organizationName" -> organizationName) + .withTokenFromContext .getWithJsonResponse[List[UnfinishedUpload]] } yield unfinishedUploads - def reportUpload(dataSourceId: DataSourceId, - datasetSizeBytes: Long, - needsConversion: Boolean, - viaAddRoute: Boolean, - userToken: Option[String]): Fox[Unit] = + def reportUpload(dataSourceId: DataSourceId, datasetSizeBytes: Long, needsConversion: Boolean, viaAddRoute: Boolean)( + implicit tc: TokenContext): Fox[Unit] = for { _ <- rpc(s"$webknossosUri/api/datastores/$dataStoreName/reportDatasetUpload") .addQueryString("key" -> dataStoreKey) @@ -90,7 +87,7 @@ class DSRemoteWebknossosClient @Inject()( .addQueryString("needsConversion" -> needsConversion.toString) .addQueryString("viaAddRoute" -> viaAddRoute.toString) .addQueryString("datasetSizeBytes" -> datasetSizeBytes.toString) - .addQueryStringOptional("token", userToken) + .withTokenFromContext .post() } yield () @@ -100,12 +97,11 @@ class DSRemoteWebknossosClient @Inject()( .silent .put(dataSources) - def reserveDataSourceUpload(info: ReserveUploadInformation, userTokenOpt: Option[String]): Fox[Unit] = + def reserveDataSourceUpload(info: ReserveUploadInformation)(implicit tc: TokenContext): Fox[Unit] = for { - userToken <- option2Fox(userTokenOpt) ?~> "reserveUpload.noUserToken" _ <- rpc(s"$webknossosUri/api/datastores/$dataStoreName/reserveUpload") .addQueryString("key" -> dataStoreKey) - .addQueryString("token" -> userToken) + .withTokenFromContext .post(info) } yield () @@ -118,10 +114,10 @@ class DSRemoteWebknossosClient @Inject()( .addQueryString("key" -> dataStoreKey) .getWithJsonResponse[JobExportProperties] - override def requestUserAccess(userToken: Option[String], accessRequest: UserAccessRequest): Fox[UserAccessAnswer] = + override def requestUserAccess(accessRequest: UserAccessRequest)(implicit tc: TokenContext): Fox[UserAccessAnswer] = rpc(s"$webknossosUri/api/datastores/$dataStoreName/validateUserAccess") .addQueryString("key" -> dataStoreKey) - .addQueryStringOptional("token", userToken) + .withTokenFromContext .postJsonWithJsonResponse[UserAccessRequest, UserAccessAnswer](accessRequest) private lazy val tracingstoreUriCache: AlfuCache[String, String] = AlfuCache() @@ -141,13 +137,13 @@ class DSRemoteWebknossosClient @Inject()( private lazy val annotationSourceCache: AlfuCache[(String, Option[String]), AnnotationSource] = AlfuCache(timeToLive = 5 seconds, timeToIdle = 5 seconds) - def getAnnotationSource(accessToken: String, userToken: Option[String]): Fox[AnnotationSource] = + def getAnnotationSource(accessToken: String)(implicit tc: TokenContext): Fox[AnnotationSource] = annotationSourceCache.getOrLoad( - (accessToken, userToken), + (accessToken, tc.userTokenOpt), _ => rpc(s"$webknossosUri/api/annotations/source/$accessToken") .addQueryString("key" -> dataStoreKey) - .addQueryStringOptional("userToken", userToken) + .withTokenFromContext .getWithJsonResponse[AnnotationSource] ) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MeshMappingHelper.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MeshMappingHelper.scala index 391c234fc6b..14279974079 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MeshMappingHelper.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/MeshMappingHelper.scala @@ -1,5 +1,6 @@ package com.scalableminds.webknossos.datastore.services +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.tools.Fox import com.scalableminds.util.tools.Fox.{box2Fox, option2Fox} import com.scalableminds.webknossos.datastore.storage.AgglomerateFileKey @@ -22,7 +23,7 @@ trait MeshMappingHelper { agglomerateId: Long, mappingNameForMeshFile: Option[String], omitMissing: Boolean, // If true, failing lookups in the agglomerate file will just return empty list. - token: Option[String])(implicit ec: ExecutionContext): Fox[List[Long]] = + )(implicit ec: ExecutionContext, tc: TokenContext): Fox[List[Long]] = (targetMappingName, editableMappingTracingId) match { case (None, None) => // No mapping selected, assume id matches meshfile @@ -58,8 +59,7 @@ trait MeshMappingHelper { tracingstoreUri <- dsRemoteWebknossosClient.getTracingstoreUri segmentIdsResult <- dsRemoteTracingstoreClient.getEditableMappingSegmentIdsForAgglomerate(tracingstoreUri, tracingId, - agglomerateId, - token) + agglomerateId) segmentIds <- if (segmentIdsResult.agglomerateIdIsPresent) Fox.successful(segmentIdsResult.segmentIds) else // the agglomerate id is not present in the editable mapping. Fetch its info from the base mapping. diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/ComposeService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/ComposeService.scala index 26d16943db4..efed327ef37 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/ComposeService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/ComposeService.scala @@ -1,5 +1,6 @@ package com.scalableminds.webknossos.datastore.services.uploading +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.io.PathUtils import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.datastore.dataformats.layers.{ @@ -67,7 +68,7 @@ class ComposeService @Inject()(dataSourceRepository: DataSourceRepository, private def uploadDirectory(organizationId: String, name: String): Path = dataBaseDir.resolve(organizationId).resolve(name) - def composeDataset(composeRequest: ComposeRequest, userToken: Option[String]): Fox[DataSource] = + def composeDataset(composeRequest: ComposeRequest)(implicit tc: TokenContext): Fox[DataSource] = for { _ <- dataSourceService.assertDataDirWritable(composeRequest.organizationId) reserveUploadInfo = ReserveUploadInformation("", @@ -78,7 +79,7 @@ class ComposeService @Inject()(dataSourceRepository: DataSourceRepository, None, List(), Some(composeRequest.targetFolderId)) - _ <- remoteWebknossosClient.reserveDataSourceUpload(reserveUploadInfo, userToken) ?~> "Failed to reserve upload." + _ <- remoteWebknossosClient.reserveDataSourceUpload(reserveUploadInfo) ?~> "Failed to reserve upload." directory = uploadDirectory(composeRequest.organizationId, composeRequest.newDatasetName) _ = PathUtils.ensureDirectory(directory) dataSource <- createDatasource(composeRequest, composeRequest.organizationId) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala index 564e51d671a..15f8dd30475 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteDatastoreClient.scala @@ -1,6 +1,7 @@ package com.scalableminds.webknossos.tracingstore import com.google.inject.Inject +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.geometry.Vec3Int import com.scalableminds.util.tools.Fox @@ -38,36 +39,28 @@ class TSRemoteDatastoreClient @Inject()( private lazy val largestAgglomerateIdCache: AlfuCache[(RemoteFallbackLayer, String, Option[String]), Long] = AlfuCache(timeToLive = 10 minutes) - def getAgglomerateSkeleton(userToken: Option[String], - remoteFallbackLayer: RemoteFallbackLayer, - mappingName: String, - agglomerateId: Long): Fox[Array[Byte]] = + def getAgglomerateSkeleton(remoteFallbackLayer: RemoteFallbackLayer, mappingName: String, agglomerateId: Long)( + implicit tc: TokenContext): Fox[Array[Byte]] = for { remoteLayerUri <- getRemoteLayerUri(remoteFallbackLayer) - result <- rpc(s"$remoteLayerUri/agglomerates/$mappingName/skeleton/$agglomerateId") - .addQueryStringOptional("token", userToken) - .getWithBytesResponse + result <- rpc(s"$remoteLayerUri/agglomerates/$mappingName/skeleton/$agglomerateId").withTokenFromContext.getWithBytesResponse } yield result - def getData(remoteFallbackLayer: RemoteFallbackLayer, - dataRequests: List[WebknossosDataRequest], - userToken: Option[String]): Fox[(Array[Byte], List[Int])] = + def getData(remoteFallbackLayer: RemoteFallbackLayer, dataRequests: List[WebknossosDataRequest])( + implicit tc: TokenContext): Fox[(Array[Byte], List[Int])] = for { remoteLayerUri <- getRemoteLayerUri(remoteFallbackLayer) - response <- rpc(s"$remoteLayerUri/data").addQueryStringOptional("token", userToken).silent.post(dataRequests) + response <- rpc(s"$remoteLayerUri/data").withTokenFromContext.silent.post(dataRequests) _ <- bool2Fox(Status.isSuccessful(response.status)) bytes = response.bodyAsBytes.toArray indices <- parseMissingBucketHeader(response.header(missingBucketsHeader)) ?~> "failed to parse missing bucket header" } yield (bytes, indices) - def getVoxelAtPosition(userToken: Option[String], - remoteFallbackLayer: RemoteFallbackLayer, - pos: Vec3Int, - mag: Vec3Int): Fox[Array[Byte]] = + def getVoxelAtPosition(remoteFallbackLayer: RemoteFallbackLayer, pos: Vec3Int, mag: Vec3Int)( + implicit tc: TokenContext): Fox[Array[Byte]] = for { remoteLayerUri <- getRemoteLayerUri(remoteFallbackLayer) - result <- rpc(s"$remoteLayerUri/data") - .addQueryStringOptional("token", userToken) + result <- rpc(s"$remoteLayerUri/data").withTokenFromContext .addQueryString("x" -> pos.x.toString) .addQueryString("y" -> pos.y.toString) .addQueryString("z" -> pos.z.toString) @@ -81,33 +74,25 @@ class TSRemoteDatastoreClient @Inject()( def getAgglomerateIdsForSegmentIds(remoteFallbackLayer: RemoteFallbackLayer, mappingName: String, - segmentIdsOrdered: List[Long], - userToken: Option[String]): Fox[List[Long]] = + segmentIdsOrdered: List[Long])(implicit tc: TokenContext): Fox[List[Long]] = for { remoteLayerUri <- getRemoteLayerUri(remoteFallbackLayer) segmentIdsOrderedProto = ListOfLong(items = segmentIdsOrdered) - result <- rpc(s"$remoteLayerUri/agglomerates/$mappingName/agglomeratesForSegments") - .addQueryStringOptional("token", userToken) - .silent + result <- rpc(s"$remoteLayerUri/agglomerates/$mappingName/agglomeratesForSegments").withTokenFromContext.silent .postProtoWithProtoResponse[ListOfLong, ListOfLong](segmentIdsOrderedProto)(ListOfLong) } yield result.items.toList - def getAgglomerateGraph(remoteFallbackLayer: RemoteFallbackLayer, - baseMappingName: String, - agglomerateId: Long, - userToken: Option[String]): Fox[AgglomerateGraph] = + def getAgglomerateGraph(remoteFallbackLayer: RemoteFallbackLayer, baseMappingName: String, agglomerateId: Long)( + implicit tc: TokenContext): Fox[AgglomerateGraph] = for { remoteLayerUri <- getRemoteLayerUri(remoteFallbackLayer) - result <- rpc(s"$remoteLayerUri/agglomerates/$baseMappingName/agglomerateGraph/$agglomerateId").silent - .addQueryStringOptional("token", userToken) - .silent + result <- rpc(s"$remoteLayerUri/agglomerates/$baseMappingName/agglomerateGraph/$agglomerateId").silent.withTokenFromContext.silent .getWithProtoResponse[AgglomerateGraph](AgglomerateGraph) } yield result - def getLargestAgglomerateId(remoteFallbackLayer: RemoteFallbackLayer, - mappingName: String, - userToken: Option[String]): Fox[Long] = { - val cacheKey = (remoteFallbackLayer, mappingName, userToken) + def getLargestAgglomerateId(remoteFallbackLayer: RemoteFallbackLayer, mappingName: String)( + implicit tc: TokenContext): Fox[Long] = { + val cacheKey = (remoteFallbackLayer, mappingName, tc.userTokenOpt) largestAgglomerateIdCache.getOrLoad( cacheKey, k => @@ -121,26 +106,20 @@ class TSRemoteDatastoreClient @Inject()( ) } - def hasSegmentIndexFile(remoteFallbackLayer: RemoteFallbackLayer, userToken: Option[String]): Fox[Boolean] = + def hasSegmentIndexFile(remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[Boolean] = for { remoteLayerUri <- getRemoteLayerUri(remoteFallbackLayer) - hasIndexFile <- rpc(s"$remoteLayerUri/hasSegmentIndex") - .addQueryStringOptional("token", userToken) - .silent - .getWithJsonResponse[Boolean] + hasIndexFile <- rpc(s"$remoteLayerUri/hasSegmentIndex").withTokenFromContext.silent.getWithJsonResponse[Boolean] } yield hasIndexFile def querySegmentIndex(remoteFallbackLayer: RemoteFallbackLayer, segmentId: Long, mag: Vec3Int, mappingName: Option[String], // should be the baseMappingName in case of editable mappings - editableMappingTracingId: Option[String], - userToken: Option[String]): Fox[Seq[Vec3Int]] = + editableMappingTracingId: Option[String])(implicit tc: TokenContext): Fox[Seq[Vec3Int]] = for { remoteLayerUri <- getRemoteLayerUri(remoteFallbackLayer) - positions <- rpc(s"$remoteLayerUri/segmentIndex/$segmentId") - .addQueryStringOptional("token", userToken) - .silent + positions <- rpc(s"$remoteLayerUri/segmentIndex/$segmentId").withTokenFromContext.silent .postJsonWithJsonResponse[GetSegmentIndexParameters, Seq[Vec3Int]](GetSegmentIndexParameters( mag, cubeSize = Vec3Int.ones, // Don't use the cubeSize parameter here (since we want to calculate indices later anyway) @@ -157,13 +136,10 @@ class TSRemoteDatastoreClient @Inject()( segmentIds: Seq[Long], mag: Vec3Int, mappingName: Option[String], // should be the baseMappingName in case of editable mappings - editableMappingTracingId: Option[String], - userToken: Option[String]): Fox[Seq[(Long, Seq[Vec3Int])]] = + editableMappingTracingId: Option[String])(implicit tc: TokenContext): Fox[Seq[(Long, Seq[Vec3Int])]] = for { remoteLayerUri <- getRemoteLayerUri(remoteFallbackLayer) - result <- rpc(s"$remoteLayerUri/segmentIndex") - .addQueryStringOptional("token", userToken) - .silent + result <- rpc(s"$remoteLayerUri/segmentIndex").withTokenFromContext.silent .postJsonWithJsonResponse[GetMultipleSegmentIndexParameters, Seq[SegmentIndexData]]( GetMultipleSegmentIndexParameters(segmentIds.toList, mag, @@ -173,25 +149,22 @@ class TSRemoteDatastoreClient @Inject()( } yield result.map(data => (data.segmentId, data.positions)) - def loadFullMeshStl(token: Option[String], - remoteFallbackLayer: RemoteFallbackLayer, - fullMeshRequest: FullMeshRequest): Fox[Array[Byte]] = + def loadFullMeshStl(remoteFallbackLayer: RemoteFallbackLayer, fullMeshRequest: FullMeshRequest)( + implicit tc: TokenContext): Fox[Array[Byte]] = for { remoteLayerUri <- getRemoteLayerUri(remoteFallbackLayer) - result <- rpc(s"$remoteLayerUri/meshes/fullMesh.stl") - .addQueryStringOptional("token", token) + result <- rpc(s"$remoteLayerUri/meshes/fullMesh.stl").withTokenFromContext .postJsonWithBytesResponse(fullMeshRequest) } yield result - def voxelSizeForTracingWithCache(tracingId: String, token: Option[String]): Fox[VoxelSize] = - voxelSizeCache.getOrLoad(tracingId, tId => voxelSizeForTracing(tId, token)) + def voxelSizeForTracingWithCache(tracingId: String)(implicit tc: TokenContext): Fox[VoxelSize] = + voxelSizeCache.getOrLoad(tracingId, tId => voxelSizeForTracing(tId)) - private def voxelSizeForTracing(tracingId: String, token: Option[String]): Fox[VoxelSize] = + private def voxelSizeForTracing(tracingId: String)(implicit tc: TokenContext): Fox[VoxelSize] = for { dataSourceId <- remoteWebknossosClient.getDataSourceIdForTracing(tracingId) dataStoreUri <- dataStoreUriWithCache(dataSourceId.team, dataSourceId.name) - result <- rpc(s"$dataStoreUri/data/datasets/${dataSourceId.team}/${dataSourceId.name}/readInboxDataSource") - .addQueryStringOptional("token", token) + result <- rpc(s"$dataStoreUri/data/datasets/${dataSourceId.team}/${dataSourceId.name}/readInboxDataSource").withTokenFromContext .getWithJsonResponse[InboxDataSource] scale <- result.voxelSizeOpt ?~> "could not determine voxel size of dataset" } yield scale diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala index f1fd9555f4b..09e3e019e8c 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala @@ -1,6 +1,7 @@ package com.scalableminds.webknossos.tracingstore import com.google.inject.Inject +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox @@ -73,10 +74,10 @@ class TSRemoteWebknossosClient @Inject()( .getWithJsonResponse[DataSourceId] ) - override def requestUserAccess(token: Option[String], accessRequest: UserAccessRequest): Fox[UserAccessAnswer] = + override def requestUserAccess(accessRequest: UserAccessRequest)(implicit tc: TokenContext): Fox[UserAccessAnswer] = rpc(s"$webknossosUri/api/tracingstores/$tracingStoreName/validateUserAccess") .addQueryString("key" -> tracingStoreKey) - .addQueryStringOptional("token", token) + .withTokenFromContext .postJsonWithJsonResponse[UserAccessRequest, UserAccessAnswer](accessRequest) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala index 0738dd3c8c3..63e700c8bcf 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala @@ -1,5 +1,6 @@ package com.scalableminds.webknossos.tracingstore.annotation +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.tools.{Fox, JsonHelper} import com.scalableminds.util.tools.Fox.bool2Fox import com.scalableminds.webknossos.tracingstore.TracingStoreRedisStore @@ -61,16 +62,16 @@ class AnnotationTransactionService @Inject()( Some(expiry)) } yield () - private def handleUpdateGroupForTransaction(annotationId: String, - previousVersionFox: Fox[Long], - updateGroup: UpdateActionGroup, - userToken: Option[String])(implicit ec: ExecutionContext): Fox[Long] = + private def handleUpdateGroupForTransaction( + annotationId: String, + previousVersionFox: Fox[Long], + updateGroup: UpdateActionGroup)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Long] = for { previousCommittedVersion: Long <- previousVersionFox result <- if (previousCommittedVersion + 1 == updateGroup.version) { if (updateGroup.transactionGroupCount == updateGroup.transactionGroupIndex + 1) { // Received the last group of this transaction - commitWithPending(annotationId, updateGroup, userToken) + commitWithPending(annotationId, updateGroup) } else { for { _ <- saveUncommitted(annotationId, @@ -92,15 +93,15 @@ class AnnotationTransactionService @Inject()( // For an update group (that is the last of a transaction), fetch all previous uncommitted for the same transaction // and commit them all. - private def commitWithPending(annotationId: String, updateGroup: UpdateActionGroup, userToken: Option[String])( - implicit ec: ExecutionContext): Fox[Long] = + private def commitWithPending(annotationId: String, updateGroup: UpdateActionGroup)(implicit ec: ExecutionContext, + tc: TokenContext): Fox[Long] = for { previousActionGroupsToCommit <- getAllUncommittedFor(annotationId, updateGroup.transactionId) _ <- bool2Fox( previousActionGroupsToCommit .exists(_.transactionGroupIndex == 0) || updateGroup.transactionGroupCount == 1) ?~> s"Trying to commit a transaction without a group that has transactionGroupIndex 0." concatenatedGroup = concatenateUpdateGroupsOfTransaction(previousActionGroupsToCommit, updateGroup) - commitResult <- commitUpdates(annotationId, List(concatenatedGroup), userToken) + commitResult <- commitUpdates(annotationId, List(concatenatedGroup)) _ <- removeAllUncommittedFor(annotationId, updateGroup.transactionId) } yield commitResult @@ -146,22 +147,22 @@ class AnnotationTransactionService @Inject()( ) } - def handleUpdateGroups(annotationId: String, updateGroups: List[UpdateActionGroup], userToken: Option[String])( - implicit ec: ExecutionContext): Fox[Long] = + def handleUpdateGroups(annotationId: String, updateGroups: List[UpdateActionGroup])(implicit ec: ExecutionContext, + tc: TokenContext): Fox[Long] = if (updateGroups.forall(_.transactionGroupCount == 1)) { - commitUpdates(annotationId, updateGroups, userToken) + commitUpdates(annotationId, updateGroups) } else { updateGroups.foldLeft(annotationService.currentMaterializableVersion(annotationId)) { (currentCommittedVersionFox, updateGroup) => - handleUpdateGroupForTransaction(annotationId, currentCommittedVersionFox, updateGroup, userToken) + handleUpdateGroupForTransaction(annotationId, currentCommittedVersionFox, updateGroup) } } // Perform version check and commit the passed updates - private def commitUpdates(annotationId: String, updateGroups: List[UpdateActionGroup], userToken: Option[String])( - implicit ec: ExecutionContext): Fox[Long] = + private def commitUpdates(annotationId: String, updateGroups: List[UpdateActionGroup])(implicit ec: ExecutionContext, + tc: TokenContext): Fox[Long] = for { - _ <- annotationService.reportUpdates(annotationId, updateGroups, userToken) + _ <- annotationService.reportUpdates(annotationId, updateGroups) currentCommittedVersion: Fox[Long] = annotationService.currentMaterializableVersion(annotationId) _ = logger.info(s"trying to commit ${updateGroups .map(_.actions.length) @@ -170,7 +171,7 @@ class AnnotationTransactionService @Inject()( previousVersion.flatMap { prevVersion: Long => if (prevVersion + 1 == updateGroup.version) { for { - _ <- handleUpdateGroup(annotationId, updateGroup, userToken) + _ <- handleUpdateGroup(annotationId, updateGroup) _ <- saveToHandledGroupIdStore(annotationId, updateGroup.transactionId, updateGroup.version, @@ -181,15 +182,15 @@ class AnnotationTransactionService @Inject()( } } yield newVersion - private def handleUpdateGroup(annotationId: String, updateActionGroup: UpdateActionGroup, userToken: Option[String])( - implicit ec: ExecutionContext): Fox[Unit] = + private def handleUpdateGroup(annotationId: String, updateActionGroup: UpdateActionGroup)( + implicit ec: ExecutionContext, + tc: TokenContext): Fox[Unit] = for { updateActionsJson <- Fox.successful(Json.toJson(preprocessActionsForStorage(updateActionGroup))) _ <- tracingDataStore.annotationUpdates.put(annotationId, updateActionGroup.version, updateActionsJson) bucketMutatingActions = findBucketMutatingActions(updateActionGroup) _ <- Fox.runIf(bucketMutatingActions.nonEmpty)( - volumeTracingService - .applyBucketMutatingActions(annotationId, bucketMutatingActions, updateActionGroup.version, userToken)) + volumeTracingService.applyBucketMutatingActions(annotationId, bucketMutatingActions, updateActionGroup.version)) } yield () private def findBucketMutatingActions(updateActionGroup: UpdateActionGroup): List[BucketMutatingVolumeUpdateAction] = diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index d1dd18cf044..2a1eda1d2dc 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -39,6 +39,11 @@ case class AnnotationWithTracings( } } yield volumeTracing + def getEditableMappingInfo(tracingId: String): Box[EditableMappingInfo] = + for { + (info, _) <- editableMappingsByTracingId.get(tracingId) + } yield info + def version: Long = annotation.version def addTracing(a: AddLayerAnnotationUpdateAction): AnnotationWithTracings = diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index eb46397ca28..c0b5b191620 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -1,5 +1,6 @@ package com.scalableminds.webknossos.tracingstore.annotation +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox import com.scalableminds.util.tools.Fox.option2Fox @@ -20,7 +21,6 @@ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ ApplyableVolumeUpdateAction, BucketMutatingVolumeUpdateAction, - VolumeTracingService, VolumeUpdateAction } import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} @@ -37,7 +37,7 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl extends KeyValueStoreImplicits with LazyLogging { - def reportUpdates(annotationId: String, updateGroups: List[UpdateActionGroup], userToken: Option[String]): Fox[Unit] = + def reportUpdates(annotationId: String, updateGroups: List[UpdateActionGroup])(implicit tc: TokenContext): Fox[Unit] = for { _ <- remoteWebknossosClient.reportTracingUpdates( TracingUpdatesReport( @@ -46,7 +46,7 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl statistics = updateGroups.flatMap(_.stats).lastOption, // TODO statistics per tracing/layer significantChangesCount = updateGroups.map(_.significantChangesCount).sum, viewChangesCount = updateGroups.map(_.viewChangesCount).sum, - userToken + tc.userTokenOpt )) } yield () @@ -115,17 +115,17 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl } yield Json.toJson(updateActionGroupsJs) } - def get(annotationId: String, version: Option[Long], userToken: Option[String])( - implicit ec: ExecutionContext): Fox[AnnotationProto] = + def get(annotationId: String, version: Option[Long])(implicit ec: ExecutionContext, + tc: TokenContext): Fox[AnnotationProto] = for { - withTracings <- getWithTracings(annotationId, version, List.empty, List.empty, userToken) + withTracings <- getWithTracings(annotationId, version, List.empty, List.empty) } yield withTracings.annotation def getWithTracings(annotationId: String, version: Option[Long], requestedSkeletonTracingIds: List[String], - requestedVolumeTracingIds: List[String], - userToken: Option[String])(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = + requestedVolumeTracingIds: List[String])(implicit ec: ExecutionContext, + tc: TokenContext): Fox[AnnotationWithTracings] = for { annotationWithVersion <- tracingDataStore.annotations.get(annotationId, version)(fromProtoBytes[AnnotationProto]) ?~> "getAnnotation.failed" annotation = annotationWithVersion.value @@ -133,17 +133,16 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl annotationId, version, requestedSkeletonTracingIds, - requestedVolumeTracingIds, - userToken) ?~> "applyUpdates.failed" + requestedVolumeTracingIds) ?~> "applyUpdates.failed" } yield updated - private def applyPendingUpdates( - annotation: AnnotationProto, - annotationId: String, - targetVersionOpt: Option[Long], - requestedSkeletonTracingIds: List[String], - requestedVolumeTracingIds: List[String], - userToken: Option[String])(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = + private def applyPendingUpdates(annotation: AnnotationProto, + annotationId: String, + targetVersionOpt: Option[Long], + requestedSkeletonTracingIds: List[String], + requestedVolumeTracingIds: List[String])( + implicit ec: ExecutionContext, + tc: TokenContext): Fox[AnnotationWithTracings] = for { targetVersion <- determineTargetVersion(annotation, annotationId, targetVersionOpt) ?~> "determineTargetVersion.failed" updates <- findPendingUpdates(annotationId, annotation.version, targetVersion) ?~> "findPendingUpdates.failed" @@ -151,7 +150,7 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl updates, requestedSkeletonTracingIds, requestedVolumeTracingIds) ?~> "findTracingsForUpdates.failed" - updated <- applyUpdates(annotationWithTracings, annotationId, updates, targetVersion, userToken) ?~> "applyUpdates.inner.failed" + updated <- applyUpdates(annotationWithTracings, annotationId, updates, targetVersion) ?~> "applyUpdates.inner.failed" } yield updated private def findTracingsForUpdates( @@ -189,11 +188,11 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl } yield AnnotationWithTracings(annotation, skeletonTracingsMap ++ volumeTracingsMap, editableMappingsMap) } - private def applyUpdates(annotation: AnnotationWithTracings, - annotationId: String, - updates: List[UpdateAction], - targetVersion: Long, - userToken: Option[String])(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = { + private def applyUpdates( + annotation: AnnotationWithTracings, + annotationId: String, + updates: List[UpdateAction], + targetVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = { def updateIter(annotationWithTracingsFox: Fox[AnnotationWithTracings], remainingUpdates: List[UpdateAction]): Fox[AnnotationWithTracings] = diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala index 6bb280fe104..9b9f9f191a9 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala @@ -38,7 +38,7 @@ class SkeletonTracingController @Inject()(val tracingService: SkeletonTracingSer def mergedFromContents(token: Option[String], persist: Boolean): Action[SkeletonTracings] = Action.async(validateProto[SkeletonTracings]) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.webknossos) { val tracings: List[Option[SkeletonTracing]] = request.body for { mergedTracing <- Fox.box2Fox(tracingService.merge(tracings.flatten, MergedVolumeStats.empty(), Empty)) @@ -59,13 +59,9 @@ class SkeletonTracingController @Inject()(val tracingService: SkeletonTracingSer boundingBox: Option[String]): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.webknossos) { for { - tracing <- tracingService.find(annotationId, - tracingId, - version, - applyUpdates = true, - userToken = urlOrHeaderToken(token, request)) ?~> Messages( + tracing <- tracingService.find(annotationId, tracingId, version, applyUpdates = true) ?~> Messages( "tracing.notFound") editPositionParsed <- Fox.runOptional(editPosition)(Vec3Int.fromUriLiteral) editRotationParsed <- Fox.runOptional(editRotation)(Vec3Double.fromUriLiteral) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index e2858994667..d398abbdc4f 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -29,7 +29,7 @@ class TSAnnotationController @Inject()( def save(token: Option[String], annotationId: String): Action[AnnotationProto] = Action.async(validateProto[AnnotationProto]) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.webknossos) { for { // TODO assert id does not already exist _ <- tracingDataStore.annotations.put(annotationId, 0L, request.body) @@ -43,12 +43,9 @@ class TSAnnotationController @Inject()( Action.async(validateJson[List[UpdateActionGroup]]) { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { - accessTokenService.validateAccess(UserAccessRequest.writeAnnotation(annotationId), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.writeAnnotation(annotationId)) { for { - _ <- annotationTransactionService.handleUpdateGroups(annotationId, - request.body, - urlOrHeaderToken(token, request)) + _ <- annotationTransactionService.handleUpdateGroups(annotationId, request.body) } yield Ok } } @@ -60,8 +57,7 @@ class TSAnnotationController @Inject()( newestVersion: Option[Long] = None, oldestVersion: Option[Long] = None): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readAnnotation(annotationId), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readAnnotation(annotationId)) { for { updateLog <- annotationService.updateActionLog(annotationId, newestVersion, oldestVersion) } yield Ok(updateLog) @@ -72,8 +68,7 @@ class TSAnnotationController @Inject()( def newestVersion(token: Option[String], annotationId: String): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readAnnotation(annotationId), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readAnnotation(annotationId)) { for { newestVersion <- annotationService.currentMaterializableVersion(annotationId) } yield JsonOk(Json.obj("version" -> newestVersion)) @@ -84,7 +79,7 @@ class TSAnnotationController @Inject()( def updateActionStatistics(token: Option[String], tracingId: String): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { statistics <- annotationService.updateActionStatistics(tracingId) } yield Ok(statistics) @@ -96,10 +91,9 @@ class TSAnnotationController @Inject()( Action.async { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { - accessTokenService.validateAccess(UserAccessRequest.readAnnotation(annotationId), - urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readAnnotation(annotationId)) { for { - annotationProto <- annotationService.get(annotationId, version, urlOrHeaderToken(token, request)) + annotationProto <- annotationService.get(annotationId, version) } yield Ok(annotationProto.toByteArray).as(protobufMimeType) } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala index d39cf16c671..133453f00b8 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala @@ -4,7 +4,6 @@ import com.scalableminds.util.tools.Fox import com.scalableminds.util.tools.JsonHelper.{boxFormat, optionFormat} import com.scalableminds.webknossos.datastore.controllers.Controller import com.scalableminds.webknossos.datastore.services.UserAccessRequest -import com.scalableminds.webknossos.tracingstore.annotation.UpdateActionGroup import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService import com.scalableminds.webknossos.tracingstore.tracings.{TracingSelector, TracingService} import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingStoreAccessTokenService} @@ -15,7 +14,6 @@ import play.api.mvc.{Action, AnyContent, PlayBodyParsers} import scalapb.{GeneratedMessage, GeneratedMessageCompanion} import scala.concurrent.ExecutionContext -import scala.concurrent.duration._ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends Controller { @@ -46,7 +44,7 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C def save(token: Option[String]): Action[T] = Action.async(validateProto[T]) { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { - accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.webknossos) { val tracing = request.body tracingService.save(tracing, None, 0).map { newId => Ok(Json.toJson(newId)) @@ -59,7 +57,7 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C def saveMultiple(token: Option[String]): Action[Ts] = Action.async(validateProto[Ts]) { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { - accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.webknossos) { val savedIds = Fox.sequence(request.body.map { tracingOpt: Option[T] => tracingOpt match { case Some(tracing) => tracingService.save(tracing, None, 0).map(Some(_)) @@ -75,13 +73,9 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C def get(token: Option[String], annotationId: String, tracingId: String, version: Option[Long]): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, - tracingId, - version, - applyUpdates = true, - userToken = urlOrHeaderToken(token, request)) ?~> Messages( + tracing <- tracingService.find(annotationId, tracingId, version, applyUpdates = true) ?~> Messages( "tracing.notFound") } yield Ok(tracing.toByteArray).as(protobufMimeType) } @@ -91,11 +85,9 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C def getMultiple(token: Option[String]): Action[List[Option[TracingSelector]]] = Action.async(validateJson[List[Option[TracingSelector]]]) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.webknossos) { for { - tracings <- tracingService.findMultiple(request.body, - applyUpdates = true, - userToken = urlOrHeaderToken(token, request)) + tracings <- tracingService.findMultiple(request.body, applyUpdates = true) } yield { Ok(tracings.toByteArray).as(protobufMimeType) } @@ -106,11 +98,9 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C def mergedFromIds(token: Option[String], persist: Boolean): Action[List[Option[TracingSelector]]] = Action.async(validateJson[List[Option[TracingSelector]]]) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.webknossos) { for { - tracingOpts <- tracingService.findMultiple(request.body, - applyUpdates = true, - userToken = urlOrHeaderToken(token, request)) ?~> Messages( + tracingOpts <- tracingService.findMultiple(request.body, applyUpdates = true) ?~> Messages( "tracing.notFound") tracingsWithIds = tracingOpts.zip(request.body).flatMap { case (Some(tracing), Some(selector)) => Some((tracing, selector.tracingId)) @@ -121,11 +111,8 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C tracingsWithIds.map(_._1), newId, newVersion = 0L, - toCache = !persist, - token) - newEditableMappingIdBox <- tracingService - .mergeEditableMappings(tracingsWithIds, urlOrHeaderToken(token, request)) - .futureBox + toCache = !persist) + newEditableMappingIdBox <- tracingService.mergeEditableMappings(tracingsWithIds).futureBox newEditableMappingIdOpt <- newEditableMappingIdBox match { case Full(newEditableMappingId) => Fox.successful(Some(newEditableMappingId)) case Empty => Fox.successful(None) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 8621dbada90..4eb936fc829 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -27,7 +27,7 @@ import com.scalableminds.webknossos.datastore.services.{ FullMeshRequest, UserAccessRequest } -import com.scalableminds.webknossos.tracingstore.annotation.UpdateActionGroup +import com.scalableminds.webknossos.tracingstore.annotation.{AnnotationTransactionService, UpdateActionGroup} import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ EditableMappingService, @@ -44,7 +44,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ VolumeSegmentStatisticsService, VolumeTracingService } -import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits} +import com.scalableminds.webknossos.tracingstore.tracings.KeyValueStoreImplicits import com.scalableminds.webknossos.tracingstore.{ TSRemoteDatastoreClient, TSRemoteWebknossosClient, @@ -67,6 +67,7 @@ class VolumeTracingController @Inject()( val config: TracingStoreConfig, val remoteDataStoreClient: TSRemoteDatastoreClient, val accessTokenService: TracingStoreAccessTokenService, + annotationTransactionService: AnnotationTransactionService, editableMappingService: EditableMappingService, val slackNotificationService: TSSlackNotificationService, val remoteWebknossosClient: TSRemoteWebknossosClient, @@ -97,14 +98,13 @@ class VolumeTracingController @Inject()( Action.async { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { - accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.webknossos) { for { initialData <- request.body.asRaw.map(_.asFile) ?~> Messages("zipFile.notFound") - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( - "tracing.notFound") + tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") resolutionRestrictions = ResolutionRestrictions(minResolution, maxResolution) resolutions <- tracingService - .initializeWithData(annotationId, tracingId, tracing, initialData, resolutionRestrictions, token) + .initializeWithData(annotationId, tracingId, tracing, initialData, resolutionRestrictions) .toFox _ <- tracingService.updateResolutionList(tracingId, tracing, resolutions) } yield Ok(Json.toJson(tracingId)) @@ -116,7 +116,7 @@ class VolumeTracingController @Inject()( def mergedFromContents(token: Option[String], persist: Boolean): Action[VolumeTracings] = Action.async(validateProto[VolumeTracings]) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.webknossos) { for { _ <- Fox.successful(()) tracings = request.body @@ -136,13 +136,12 @@ class VolumeTracingController @Inject()( Action.async { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { - accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.webknossos) { for { initialData <- request.body.asRaw.map(_.asFile) ?~> Messages("zipFile.notFound") - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( - "tracing.notFound") + tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") resolutions <- tracingService - .initializeWithDataMultiple(annotationId, tracingId, tracing, initialData, token) + .initializeWithDataMultiple(annotationId, tracingId, tracing, initialData) .toFox _ <- tracingService.updateResolutionList(tracingId, tracing, resolutions) } yield Ok(Json.toJson(tracingId)) @@ -160,13 +159,9 @@ class VolumeTracingController @Inject()( voxelSizeUnit: Option[String]): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, - tracingId, - version, - userToken = urlOrHeaderToken(token, request)) ?~> Messages( - "tracing.notFound") + tracing <- tracingService.find(annotationId, tracingId, version) ?~> Messages("tracing.notFound") volumeDataZipFormatParsed <- VolumeDataZipFormat.fromString(volumeDataZipFormat).toFox voxelSizeFactorParsedOpt <- Fox.runOptional(voxelSizeFactor)(Vec3Double.fromUriLiteral) voxelSizeUnitParsedOpt <- Fox.runOptional(voxelSizeUnit)(LengthUnit.fromString) @@ -186,12 +181,11 @@ class VolumeTracingController @Inject()( def data(token: Option[String], annotationId: String, tracingId: String): Action[List[WebknossosDataRequest]] = Action.async(validateJson[List[WebknossosDataRequest]]) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( - "tracing.notFound") + tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") (data, indices) <- if (tracing.getHasEditableMapping) - editableMappingService.volumeData(tracing, tracingId, request.body, urlOrHeaderToken(token, request)) + editableMappingService.volumeData(tracing, tracingId, request.body) else tracingService.data(tracingId, tracing, request.body) } yield Ok(data).withHeaders(getMissingBucketsHeaders(indices): _*) } @@ -216,11 +210,9 @@ class VolumeTracingController @Inject()( boundingBox: Option[String]): Action[AnyContent] = Action.async { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { - val userToken = urlOrHeaderToken(token, request) - accessTokenService.validateAccess(UserAccessRequest.webknossos, userToken) { + accessTokenService.validateAccess(UserAccessRequest.webknossos) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = userToken) ?~> Messages( - "tracing.notFound") + tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") _ = logger.info(s"Duplicating volume tracing $tracingId...") datasetBoundingBox = request.body.asJson.flatMap(_.validateOpt[BoundingBox].asOpt.flatten) resolutionRestrictions = ResolutionRestrictions(minResolution, maxResolution) @@ -230,7 +222,7 @@ class VolumeTracingController @Inject()( remoteFallbackLayerOpt <- Fox.runIf(tracing.getHasEditableMapping)( tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId)) newEditableMappingId <- Fox.runIf(tracing.getHasEditableMapping)( - editableMappingService.duplicate(tracing.mappingName, version = None, remoteFallbackLayerOpt, userToken)) + editableMappingService.duplicate(tracing.mappingName, version = None, remoteFallbackLayerOpt)) (newId, newTracing) <- tracingService.duplicate( annotationId, tracingId, @@ -241,11 +233,9 @@ class VolumeTracingController @Inject()( editPositionParsed, editRotationParsed, boundingBoxParsed, - newEditableMappingId, - userToken + newEditableMappingId ) - _ <- Fox.runIfOptionTrue(downsample)( - tracingService.downsample(annotationId, newId, tracingId, newTracing, userToken)) + _ <- Fox.runIfOptionTrue(downsample)(tracingService.downsample(annotationId, newId, tracingId, newTracing)) } yield Ok(Json.toJson(newId)) } } @@ -257,18 +247,16 @@ class VolumeTracingController @Inject()( tracingId: String): Action[MultipartFormData[TemporaryFile]] = Action.async(parse.multipartFormData) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.writeTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.writeTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( - "tracing.notFound") + tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") currentVersion <- request.body.dataParts("currentVersion").headOption.flatMap(_.toIntOpt).toFox zipFile <- request.body.files.headOption.map(f => new File(f.ref.path.toString)).toFox largestSegmentId <- tracingService.importVolumeData(annotationId, tracingId, tracing, zipFile, - currentVersion, - urlOrHeaderToken(token, request)) + currentVersion) } yield Ok(Json.toJson(largestSegmentId)) } } @@ -280,20 +268,14 @@ class VolumeTracingController @Inject()( dryRun: Boolean): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.webknossos, urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.webknossos) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( - "tracing.notFound") + tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") currentVersion <- tracingService.currentVersion(tracingId) before = Instant.now - canAddSegmentIndex <- tracingService.checkIfSegmentIndexMayBeAdded(tracingId, tracing, token) - processedBucketCountOpt <- Fox.runIf(canAddSegmentIndex)( - tracingService.addSegmentIndex(annotationId, - tracingId, - tracing, - currentVersion, - urlOrHeaderToken(token, request), - dryRun)) ?~> "addSegmentIndex.failed" + canAddSegmentIndex <- tracingService.checkIfSegmentIndexMayBeAdded(tracingId, tracing) + processedBucketCountOpt <- Fox.runIf(canAddSegmentIndex)(tracingService + .addSegmentIndex(annotationId, tracingId, tracing, currentVersion, dryRun)) ?~> "addSegmentIndex.failed" currentVersionNew <- tracingService.currentVersion(tracingId) _ <- Fox.runIf(!dryRun)(bool2Fox( processedBucketCountOpt.isEmpty || currentVersionNew == currentVersion + 1L) ?~> "Version increment failed. Looks like someone edited the annotation layer in the meantime.") @@ -312,7 +294,7 @@ class VolumeTracingController @Inject()( newestVersion: Option[Long] = None, oldestVersion: Option[Long] = None): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { updateLog <- tracingService.updateActionLog(tracingId, newestVersion, oldestVersion) } yield Ok(updateLog) @@ -324,16 +306,15 @@ class VolumeTracingController @Inject()( annotationId: String, tracingId: String): Action[WebknossosAdHocMeshRequest] = Action.async(validateJson[WebknossosAdHocMeshRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { // The client expects the ad-hoc mesh as a flat float-array. Three consecutive floats form a 3D point, three // consecutive 3D points (i.e., nine floats) form a triangle. // There are no shared vertices between triangles. - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( - "tracing.notFound") + tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") (vertices, neighbors) <- if (tracing.getHasEditableMapping) - editableMappingService.createAdHocMesh(tracing, tracingId, request.body, urlOrHeaderToken(token, request)) - else tracingService.createAdHocMesh(annotationId, tracingId, request.body, urlOrHeaderToken(token, request)) + editableMappingService.createAdHocMesh(tracing, tracingId, request.body) + else tracingService.createAdHocMesh(annotationId, tracingId, request.body) } yield { // We need four bytes for each float val responseBuffer = ByteBuffer.allocate(vertices.length * 4).order(ByteOrder.LITTLE_ENDIAN) @@ -345,9 +326,9 @@ class VolumeTracingController @Inject()( def loadFullMeshStl(token: Option[String], annotationId: String, tracingId: String): Action[FullMeshRequest] = Action.async(validateJson[FullMeshRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - data: Array[Byte] <- fullMeshService.loadFor(token: Option[String], annotationId, tracingId, request.body) ?~> "mesh.file.loadChunk.failed" + data: Array[Byte] <- fullMeshService.loadFor(annotationId, tracingId, request.body) ?~> "mesh.file.loadChunk.failed" } yield Ok(data) } } @@ -360,9 +341,9 @@ class VolumeTracingController @Inject()( def findData(token: Option[String], annotationId: String, tracingId: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - positionOpt <- tracingService.findData(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) + positionOpt <- tracingService.findData(annotationId, tracingId) } yield { Ok(Json.obj("position" -> positionOpt, "resolution" -> positionOpt.map(_ => Vec3Int.ones))) } @@ -374,17 +355,15 @@ class VolumeTracingController @Inject()( tracingId: String, agglomerateId: Long): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) + tracing <- tracingService.find(annotationId, tracingId) _ <- bool2Fox(tracing.getHasEditableMapping) ?~> "Cannot query agglomerate skeleton for volume annotation" mappingName <- tracing.mappingName ?~> "annotation.agglomerateSkeleton.noMappingSet" remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - agglomerateSkeletonBytes <- editableMappingService.getAgglomerateSkeletonWithFallback( - mappingName, - remoteFallbackLayer, - agglomerateId, - urlOrHeaderToken(token, request)) + agglomerateSkeletonBytes <- editableMappingService.getAgglomerateSkeletonWithFallback(mappingName, + remoteFallbackLayer, + agglomerateId) } yield Ok(agglomerateSkeletonBytes) } } @@ -392,9 +371,9 @@ class VolumeTracingController @Inject()( def makeMappingEditable(token: Option[String], annotationId: String, tracingId: String): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) + tracing <- tracingService.find(annotationId, tracingId) tracingMappingName <- tracing.mappingName ?~> "annotation.noMappingSet" _ <- assertMappingIsNotLocked(tracing) _ <- bool2Fox(tracingService.volumeBucketsAreEmpty(tracingId)) ?~> "annotation.volumeBucketsNotEmpty" @@ -405,20 +384,20 @@ class VolumeTracingController @Inject()( isLocked = Some(true), actionTracingId = tracingId, actionTimestamp = Some(System.currentTimeMillis())) - /*_ <- tracingService.handleUpdateGroup( // TODO - tracingId, - UpdateActionGroup(tracing.version + 1, - System.currentTimeMillis(), - None, - List(volumeUpdate), - None, - None, - "dummyTransactionId", - 1, - 0), - tracing.version, - urlOrHeaderToken(token, request) - )*/ + _ <- annotationTransactionService + .handleUpdateGroups( // TODO replace this route by the update action only? address editable mappings by volume tracing id? + annotationId, + List( + UpdateActionGroup(tracing.version + 1, + System.currentTimeMillis(), + None, + List(volumeUpdate), + None, + None, + "dummyTransactionId", + 1, + 0)) + ) infoJson <- editableMappingService.infoJson(tracingId = tracingId, editableMappingId = editableMappingId, editableMappingInfo = editableMappingInfo, @@ -434,12 +413,12 @@ class VolumeTracingController @Inject()( def agglomerateGraphMinCut(token: Option[String], annotationId: String, tracingId: String): Action[MinCutParameters] = Action.async(validateJson[MinCutParameters]) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) + tracing <- tracingService.find(annotationId, tracingId) _ <- bool2Fox(tracing.getHasEditableMapping) ?~> "Mapping is not editable" remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - edges <- editableMappingService.agglomerateGraphMinCut(request.body, remoteFallbackLayer, token) + edges <- editableMappingService.agglomerateGraphMinCut(request.body, remoteFallbackLayer) } yield Ok(Json.toJson(edges)) } } @@ -450,14 +429,12 @@ class VolumeTracingController @Inject()( tracingId: String): Action[NeighborsParameters] = Action.async(validateJson[NeighborsParameters]) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) + tracing <- tracingService.find(annotationId, tracingId) _ <- bool2Fox(tracing.getHasEditableMapping) ?~> "Mapping is not editable" remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - (segmentId, edges) <- editableMappingService.agglomerateGraphNeighbors(request.body, - remoteFallbackLayer, - token) + (segmentId, edges) <- editableMappingService.agglomerateGraphNeighbors(request.body, remoteFallbackLayer) } yield Ok(Json.obj("segmentId" -> segmentId, "neighbors" -> Json.toJson(edges))) } } @@ -467,9 +444,9 @@ class VolumeTracingController @Inject()( annotationId: String, tracingId: String): Action[List[UpdateActionGroup]] = Action.async(validateJson[List[UpdateActionGroup]]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.writeTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.writeTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) + tracing <- tracingService.find(annotationId, tracingId) mappingName <- tracing.mappingName.toFox _ <- bool2Fox(tracing.getHasEditableMapping) ?~> "Mapping is not editable" currentVersion <- editableMappingService.getClosestMaterializableVersionOrZero(mappingName, None) @@ -482,15 +459,11 @@ class VolumeTracingController @Inject()( statistics = None, significantChangesCount = updateGroup.actions.length, viewChangesCount = 0, - urlOrHeaderToken(token, request) + tokenContextForRequest.userTokenOpt ) _ <- remoteWebknossosClient.reportTracingUpdates(report) remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - _ <- editableMappingService.update(mappingName, - updateGroup, - updateGroup.version, - remoteFallbackLayer, - urlOrHeaderToken(token, request)) + _ <- editableMappingService.update(mappingName, updateGroup, updateGroup.version, remoteFallbackLayer) } yield Ok } } @@ -501,15 +474,11 @@ class VolumeTracingController @Inject()( version: Option[Long]): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) + tracing <- tracingService.find(annotationId, tracingId) mappingName <- tracing.mappingName.toFox - remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - editableMappingInfo <- editableMappingService.getInfo(mappingName, - version, - remoteFallbackLayer, - urlOrHeaderToken(token, request)) + editableMappingInfo <- editableMappingService.getInfoNEW(annotationId, tracingId, version) infoJson <- editableMappingService.infoJson(tracingId = tracingId, editableMappingId = mappingName, editableMappingInfo = editableMappingInfo, @@ -524,23 +493,21 @@ class VolumeTracingController @Inject()( tracingId: String): Action[ListOfLong] = Action.async(validateProto[ListOfLong]) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) + tracing <- tracingService.find(annotationId, tracingId) editableMappingId <- tracing.mappingName.toFox remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) (editableMappingInfo, editableMappingVersion) <- editableMappingService.getInfoAndActualVersion( editableMappingId, requestedVersion = None, - remoteFallbackLayer = remoteFallbackLayer, - userToken = urlOrHeaderToken(token, request)) + remoteFallbackLayer = remoteFallbackLayer) relevantMapping: Map[Long, Long] <- editableMappingService.generateCombinedMappingForSegmentIds( request.body.items.toSet, editableMappingInfo, editableMappingVersion, editableMappingId, - remoteFallbackLayer, - urlOrHeaderToken(token, request)) + remoteFallbackLayer) agglomerateIdsSorted = relevantMapping.toSeq.sortBy(_._1).map(_._2) } yield Ok(ListOfLong(agglomerateIdsSorted).toByteArray) } @@ -553,16 +520,13 @@ class VolumeTracingController @Inject()( agglomerateId: Long): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) + tracing <- tracingService.find(annotationId, tracingId) mappingName <- tracing.mappingName.toFox remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) agglomerateGraphBox: Box[AgglomerateGraph] <- editableMappingService - .getAgglomerateGraphForId(mappingName, - agglomerateId, - remoteFallbackLayer, - urlOrHeaderToken(token, request)) + .getAgglomerateGraphForId(mappingName, agglomerateId, remoteFallbackLayer) .futureBox segmentIds <- agglomerateGraphBox match { case Full(agglomerateGraph) => Fox.successful(agglomerateGraph.segments) @@ -579,9 +543,9 @@ class VolumeTracingController @Inject()( annotationId: String, tracingId: String): Action[SegmentStatisticsParameters] = Action.async(validateJson[SegmentStatisticsParameters]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) + tracing <- tracingService.find(annotationId, tracingId) mappingName <- tracingService.baseMappingName(tracing) segmentVolumes <- Fox.serialCombined(request.body.segmentIds) { segmentId => volumeSegmentStatisticsService.getSegmentVolume(annotationId, @@ -589,8 +553,7 @@ class VolumeTracingController @Inject()( segmentId, request.body.mag, mappingName, - request.body.additionalCoordinates, - urlOrHeaderToken(token, request)) + request.body.additionalCoordinates) } } yield Ok(Json.toJson(segmentVolumes)) } @@ -600,9 +563,9 @@ class VolumeTracingController @Inject()( annotationId: String, tracingId: String): Action[SegmentStatisticsParameters] = Action.async(validateJson[SegmentStatisticsParameters]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) + tracing <- tracingService.find(annotationId, tracingId) mappingName <- tracingService.baseMappingName(tracing) segmentBoundingBoxes: List[BoundingBox] <- Fox.serialCombined(request.body.segmentIds) { segmentId => volumeSegmentStatisticsService.getSegmentBoundingBox(annotationId, @@ -610,8 +573,7 @@ class VolumeTracingController @Inject()( segmentId, request.body.mag, mappingName, - request.body.additionalCoordinates, - urlOrHeaderToken(token, request)) + request.body.additionalCoordinates) } } yield Ok(Json.toJson(segmentBoundingBoxes)) } @@ -622,10 +584,10 @@ class VolumeTracingController @Inject()( tracingId: String, segmentId: Long): Action[GetSegmentIndexParameters] = Action.async(validateJson[GetSegmentIndexParameters]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - fallbackLayer <- tracingService.getFallbackLayer(annotationId, tracingId, urlOrHeaderToken(token, request)) - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) + fallbackLayer <- tracingService.getFallbackLayer(annotationId, tracingId) + tracing <- tracingService.find(annotationId, tracingId) mappingName <- tracingService.baseMappingName(tracing) _ <- bool2Fox(DataLayer.bucketSize <= request.body.cubeSize) ?~> "cubeSize must be at least one bucket (32³)" bucketPositionsRaw: ListOfVec3IntProto <- volumeSegmentIndexService @@ -637,8 +599,7 @@ class VolumeTracingController @Inject()( additionalCoordinates = request.body.additionalCoordinates, additionalAxes = AdditionalAxis.fromProtosAsOpt(tracing.additionalAxes), mappingName = mappingName, - editableMappingTracingId = tracingService.editableMappingTracingId(tracing, tracingId), - userToken = urlOrHeaderToken(token, request) + editableMappingTracingId = tracingService.editableMappingTracingId(tracing, tracingId) ) bucketPositionsForCubeSize = bucketPositionsRaw.values .map(vec3IntFromProto) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala index ddc2b73a5f9..856e0545ecb 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala @@ -1,6 +1,7 @@ package com.scalableminds.webknossos.tracingstore.controllers import com.google.inject.Inject +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.geometry.Vec3Int import com.scalableminds.util.mvc.ExtendedController import com.scalableminds.util.tools.{Fox, FoxImplicits} @@ -61,10 +62,9 @@ class VolumeTracingZarrStreamingController @Inject()( tracingId: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( - "tracing.notFound") ~> NOT_FOUND + tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) additionalFiles = if (zarrVersion == 2) List(NgffMetadata.FILENAME_DOT_ZATTRS, NgffGroupHeader.FILENAME_DOT_ZGROUP) @@ -84,10 +84,9 @@ class VolumeTracingZarrStreamingController @Inject()( tracingId: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( - "tracing.notFound") ~> NOT_FOUND + tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto(_).toMagLiteral(allowScalar = true)) additionalFiles = if (zarrVersion == 2) List(NgffMetadata.FILENAME_DOT_ZATTRS, NgffGroupHeader.FILENAME_DOT_ZGROUP) @@ -102,10 +101,9 @@ class VolumeTracingZarrStreamingController @Inject()( mag: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( - "tracing.notFound") ~> NOT_FOUND + tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND @@ -127,10 +125,9 @@ class VolumeTracingZarrStreamingController @Inject()( mag: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( - "tracing.notFound") ~> NOT_FOUND + tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND @@ -141,10 +138,9 @@ class VolumeTracingZarrStreamingController @Inject()( def zArray(token: Option[String], annotationId: String, tracingId: String, mag: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( - "tracing.notFound") ~> NOT_FOUND + tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND @@ -176,10 +172,9 @@ class VolumeTracingZarrStreamingController @Inject()( def zarrJsonForMag(token: Option[String], annotationId: String, tracingId: String, mag: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( - "tracing.notFound") ~> NOT_FOUND + tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND @@ -223,7 +218,7 @@ class VolumeTracingZarrStreamingController @Inject()( def zGroup(token: Option[String], annotationId: String, tracingId: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { Future(Ok(Json.toJson(NgffGroupHeader(zarr_format = 2)))) } } @@ -238,10 +233,9 @@ class VolumeTracingZarrStreamingController @Inject()( annotationId: String, tracingId: String, ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( - "tracing.notFound") ~> NOT_FOUND + tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) dataSource <- remoteWebknossosClient.getDataSourceForTracing(tracingId) ~> NOT_FOUND @@ -257,10 +251,9 @@ class VolumeTracingZarrStreamingController @Inject()( annotationId: String, tracingId: String, ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( - "tracing.notFound") ~> NOT_FOUND + tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) dataSource <- remoteWebknossosClient.getDataSourceForTracing(tracingId) ~> NOT_FOUND @@ -279,10 +272,9 @@ class VolumeTracingZarrStreamingController @Inject()( tracingName: Option[String], zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( - "tracing.notFound") ~> NOT_FOUND + tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND zarrLayer = ZarrSegmentationLayer( name = tracingName.getOrElse(tracingId), @@ -305,10 +297,9 @@ class VolumeTracingZarrStreamingController @Inject()( coordinates: String): Action[AnyContent] = Action.async { implicit request => { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId), urlOrHeaderToken(token, request)) { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId, userToken = urlOrHeaderToken(token, request)) ?~> Messages( - "tracing.notFound") ~> NOT_FOUND + tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND @@ -329,7 +320,7 @@ class VolumeTracingZarrStreamingController @Inject()( additionalCoordinates = additionalCoordinates ) (data, missingBucketIndices) <- if (tracing.getHasEditableMapping) - editableMappingService.volumeData(tracing, tracingId, List(wkRequest), urlOrHeaderToken(token, request)) + editableMappingService.volumeData(tracing, tracingId, List(wkRequest)) else tracingService.data(tracingId, tracing, List(wkRequest)) dataWithFallback <- getFallbackLayerDataIfEmpty(tracing, tracingId, @@ -338,22 +329,21 @@ class VolumeTracingZarrStreamingController @Inject()( magParsed, Vec3Int(x, y, z), cubeSize, - additionalCoordinates, - urlOrHeaderToken(token, request)) ~> NOT_FOUND + additionalCoordinates) ~> NOT_FOUND } yield Ok(dataWithFallback) } } } - private def getFallbackLayerDataIfEmpty(tracing: VolumeTracing, - tracingId: String, - data: Array[Byte], - missingBucketIndices: List[Int], - mag: Vec3Int, - position: Vec3Int, - cubeSize: Int, - additionalCoordinates: Option[Seq[AdditionalCoordinate]], - urlToken: Option[String]): Fox[Array[Byte]] = + private def getFallbackLayerDataIfEmpty( + tracing: VolumeTracing, + tracingId: String, + data: Array[Byte], + missingBucketIndices: List[Int], + mag: Vec3Int, + position: Vec3Int, + cubeSize: Int, + additionalCoordinates: Option[Seq[AdditionalCoordinate]])(implicit tc: TokenContext): Fox[Array[Byte]] = if (missingBucketIndices.nonEmpty) { for { remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) ?~> "No data at coordinates, no fallback layer defined" @@ -367,8 +357,7 @@ class VolumeTracingZarrStreamingController @Inject()( additionalCoordinates = additionalCoordinates ) (fallbackData, fallbackMissingBucketIndices) <- remoteDataStoreClient.getData(remoteFallbackLayer, - List(request), - urlToken) + List(request)) _ <- bool2Fox(fallbackMissingBucketIndices.isEmpty) ?~> "No data at coordinations in fallback layer" } yield fallbackData } else Fox.successful(data) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/RemoteFallbackLayer.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/RemoteFallbackLayer.scala index e95880ae974..5d347c7a4b0 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/RemoteFallbackLayer.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/RemoteFallbackLayer.scala @@ -1,5 +1,6 @@ package com.scalableminds.webknossos.tracingstore.tracings +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.tools.Fox import com.scalableminds.util.tools.Fox.option2Fox @@ -36,10 +37,9 @@ trait FallbackDataHelper { datasetId <- remoteWebknossosClient.getDataSourceIdForTracing(tracingId) } yield RemoteFallbackLayer(datasetId.team, datasetId.name, layerName, tracing.elementClass) - def getFallbackDataFromDatastore( - remoteFallbackLayer: RemoteFallbackLayer, - dataRequests: List[WebknossosDataRequest], - userToken: Option[String])(implicit ec: ExecutionContext): Fox[(Array[Byte], List[Int])] = - fallbackDataCache.getOrLoad(FallbackDataKey(remoteFallbackLayer, dataRequests, userToken), - k => remoteDatastoreClient.getData(k.remoteFallbackLayer, k.dataRequests, k.userToken)) + def getFallbackDataFromDatastore(remoteFallbackLayer: RemoteFallbackLayer, dataRequests: List[WebknossosDataRequest])( + implicit ec: ExecutionContext, + tc: TokenContext): Fox[(Array[Byte], List[Int])] = + fallbackDataCache.getOrLoad(FallbackDataKey(remoteFallbackLayer, dataRequests, tc.userTokenOpt), + k => remoteDatastoreClient.getData(k.remoteFallbackLayer, k.dataRequests)) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala index 7250b28eb75..9d0291c6c51 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala @@ -1,13 +1,9 @@ package com.scalableminds.webknossos.tracingstore.tracings +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.tools.{Fox, FoxImplicits, JsonHelper} -import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.tracingstore.TracingStoreRedisStore -import com.scalableminds.webknossos.tracingstore.annotation.{ - AnnotationWithTracings, - TSAnnotationService, - UpdateActionGroup -} +import com.scalableminds.webknossos.tracingstore.annotation.{TSAnnotationService, UpdateActionGroup} import com.scalableminds.webknossos.tracingstore.tracings.TracingType.TracingType import com.scalableminds.webknossos.tracingstore.tracings.volume.MergedVolumeStats import com.typesafe.scalalogging.LazyLogging @@ -106,6 +102,7 @@ trait TracingService[T <: GeneratedMessage] def removeAllUncommittedFor(tracingId: String, transactionId: String): Fox[Unit] = uncommittedUpdatesStore.removeAllConditional(patternFor(tracingId, transactionId)) + /* // TODO ? add this to migration? private def migrateTracing(tracingFox: Fox[T], tracingId: String): Fox[T] = tracingMigrationService.migrateTracing(tracingFox).flatMap { case (tracing, hasChanged) => @@ -114,6 +111,7 @@ trait TracingService[T <: GeneratedMessage] else Fox.successful(tracing) } + */ def applyPendingUpdates(tracing: T, tracingId: String, targetVersion: Option[Long]): Fox[T] = Fox.successful(tracing) @@ -121,18 +119,14 @@ trait TracingService[T <: GeneratedMessage] tracingId: String, version: Option[Long] = None, useCache: Boolean = true, - applyUpdates: Boolean = false, - userToken: Option[String]): Fox[T] + applyUpdates: Boolean = false)(implicit tc: TokenContext): Fox[T] - def findMultiple(selectors: List[Option[TracingSelector]], - useCache: Boolean = true, - applyUpdates: Boolean = false, - userToken: Option[String]): Fox[List[Option[T]]] = + def findMultiple(selectors: List[Option[TracingSelector]], useCache: Boolean = true, applyUpdates: Boolean = false)( + implicit tc: TokenContext): Fox[List[Option[T]]] = Fox.combined { selectors.map { case Some(selector) => - find("dummyAnnotationid", selector.tracingId, selector.version, useCache, applyUpdates, userToken = userToken) - .map(Some(_)) + find("dummyAnnotationid", selector.tracingId, selector.version, useCache, applyUpdates).map(Some(_)) case None => Fox.successful(None) } } @@ -175,8 +169,7 @@ trait TracingService[T <: GeneratedMessage] tracings: Seq[T], newId: String, newVersion: Long, - toCache: Boolean, - userToken: Option[String])(implicit mp: MessagesProvider): Fox[MergedVolumeStats] + toCache: Boolean)(implicit mp: MessagesProvider, tc: TokenContext): Fox[MergedVolumeStats] - def mergeEditableMappings(tracingsWithIds: List[(T, String)], userToken: Option[String]): Fox[String] + def mergeEditableMappings(tracingsWithIds: List[(T, String)])(implicit tc: TokenContext): Fox[String] } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala index 47965cee726..f526a4247f2 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala @@ -1,5 +1,6 @@ package com.scalableminds.webknossos.tracingstore.tracings.editablemapping +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.geometry.{BoundingBox, Vec3Int} import com.scalableminds.util.tools.Fox @@ -37,8 +38,7 @@ class EditableMappingBucketProvider(layer: EditableMappingLayer) extends BucketP (editableMappingInfo, editableMappingVersion) <- layer.editableMappingService.getInfoAndActualVersion( editableMappingId, requestedVersion = None, - remoteFallbackLayer = remoteFallbackLayer, - userToken = layer.token) + remoteFallbackLayer = remoteFallbackLayer)(layer.tokenContext) dataRequest: WebknossosDataRequest = WebknossosDataRequest( position = Vec3Int(bucket.topLeft.mag1X, bucket.topLeft.mag1Y, bucket.topLeft.mag1Z), mag = bucket.mag, @@ -48,18 +48,17 @@ class EditableMappingBucketProvider(layer: EditableMappingLayer) extends BucketP version = None, additionalCoordinates = readInstruction.bucket.additionalCoordinates ) - (unmappedData, indices) <- layer.editableMappingService.getFallbackDataFromDatastore(remoteFallbackLayer, - List(dataRequest), - layer.token) + (unmappedData, indices) <- layer.editableMappingService + .getFallbackDataFromDatastore(remoteFallbackLayer, List(dataRequest))(ec, layer.tokenContext) _ <- bool2Fox(indices.isEmpty) unmappedDataTyped <- layer.editableMappingService.bytesToUnsignedInt(unmappedData, layer.tracing.elementClass) segmentIds = layer.editableMappingService.collectSegmentIds(unmappedDataTyped) - relevantMapping <- layer.editableMappingService.generateCombinedMappingForSegmentIds(segmentIds, - editableMappingInfo, - editableMappingVersion, - editableMappingId, - remoteFallbackLayer, - layer.token) + relevantMapping <- layer.editableMappingService.generateCombinedMappingForSegmentIds( + segmentIds, + editableMappingInfo, + editableMappingVersion, + editableMappingId, + remoteFallbackLayer)(layer.tokenContext) mappedData: Array[Byte] <- layer.editableMappingService.mapData(unmappedDataTyped, relevantMapping, layer.elementClass) @@ -72,7 +71,7 @@ case class EditableMappingLayer(name: String, resolutions: List[Vec3Int], largestSegmentId: Option[Long], elementClass: ElementClass.Value, - token: Option[String], + tokenContext: TokenContext, tracing: VolumeTracing, tracingId: String, editableMappingService: EditableMappingService) @@ -90,7 +89,7 @@ case class EditableMappingLayer(name: String, sharedChunkContentsCache: Option[AlfuCache[String, MultiArray]]): BucketProvider = new EditableMappingBucketProvider(layer = this) - override def bucketProviderCacheKey: String = s"$name-token=$token" + override def bucketProviderCacheKey: String = s"$name-token=${tokenContext.userTokenOpt}" override def mappings: Option[Set[String]] = None diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index 56039554e54..e82f75cb1e3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -1,6 +1,7 @@ package com.scalableminds.webknossos.tracingstore.tracings.editablemapping import com.google.inject.Inject +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.geometry.Vec3Int import com.scalableminds.util.time.Instant @@ -21,7 +22,7 @@ import com.scalableminds.webknossos.datastore.services.{ AdHocMeshServiceHolder, BinaryDataService } -import com.scalableminds.webknossos.tracingstore.annotation.{UpdateAction, UpdateActionGroup} +import com.scalableminds.webknossos.tracingstore.annotation.{TSAnnotationService, UpdateAction, UpdateActionGroup} import com.scalableminds.webknossos.tracingstore.tracings.volume.ReversionHelper import com.scalableminds.webknossos.tracingstore.tracings.{ FallbackDataHelper, @@ -92,6 +93,7 @@ object NodeWithPosition { class EditableMappingService @Inject()( val tracingDataStore: TracingDataStore, val adHocMeshServiceHolder: AdHocMeshServiceHolder, + annotationService: TSAnnotationService, val remoteDatastoreClient: TSRemoteDatastoreClient, val remoteWebknossosClient: TSRemoteWebknossosClient )(implicit ec: ExecutionContext) @@ -149,15 +151,11 @@ class EditableMappingService @Inject()( def duplicate(editableMappingIdOpt: Option[String], version: Option[Long], - remoteFallbackLayerBox: Box[RemoteFallbackLayer], - userToken: Option[String]): Fox[String] = + remoteFallbackLayerBox: Box[RemoteFallbackLayer])(implicit tc: TokenContext): Fox[String] = for { editableMappingId <- editableMappingIdOpt ?~> "duplicate on editable mapping without id" remoteFallbackLayer <- remoteFallbackLayerBox ?~> "duplicate on editable mapping without remote fallback layer" - editableMappingInfoAndVersion <- getInfoAndActualVersion(editableMappingId, - version, - remoteFallbackLayer, - userToken) + editableMappingInfoAndVersion <- getInfoAndActualVersion(editableMappingId, version, remoteFallbackLayer) newIdAndInfoV0 <- create(editableMappingInfoAndVersion._1.baseMappingName) newId = newIdAndInfoV0._1 newVersion = editableMappingInfoAndVersion._2 @@ -204,12 +202,17 @@ class EditableMappingService @Inject()( } yield () } - def getInfo(editableMappingId: String, - version: Option[Long] = None, - remoteFallbackLayer: RemoteFallbackLayer, - userToken: Option[String]): Fox[EditableMappingInfo] = + def getInfoNEW(annotationId: String, tracingId: String, version: Option[Long] = None)( + implicit tc: TokenContext): Fox[EditableMappingInfo] = for { - (info, _) <- getInfoAndActualVersion(editableMappingId, version, remoteFallbackLayer, userToken) + annotation <- annotationService.getWithTracings(annotationId, version, List(tracingId), List.empty) + tracing <- annotation.getEditableMappingInfo(tracingId) + } yield tracing + + def getInfo(editableMappingId: String, version: Option[Long] = None, remoteFallbackLayer: RemoteFallbackLayer)( + implicit tc: TokenContext): Fox[EditableMappingInfo] = + for { + (info, _) <- getInfoAndActualVersion(editableMappingId, version, remoteFallbackLayer) } yield info def getBaseMappingName(editableMappingId: String): Fox[Option[String]] = @@ -222,45 +225,40 @@ class EditableMappingService @Inject()( case _ => None } - def getInfoAndActualVersion(editableMappingId: String, - requestedVersion: Option[Long] = None, - remoteFallbackLayer: RemoteFallbackLayer, - userToken: Option[String]): Fox[(EditableMappingInfo, Long)] = + def getInfoAndActualVersion( + editableMappingId: String, + requestedVersion: Option[Long] = None, + remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[(EditableMappingInfo, Long)] = for { desiredVersion <- getClosestMaterializableVersionOrZero(editableMappingId, requestedVersion) materializedInfo <- materializedInfoCache.getOrLoad( (editableMappingId, desiredVersion), - _ => applyPendingUpdates(editableMappingId, desiredVersion, remoteFallbackLayer, userToken)) + _ => applyPendingUpdates(editableMappingId, desiredVersion, remoteFallbackLayer)) } yield (materializedInfo, desiredVersion) def update(editableMappingId: String, updateActionGroup: UpdateActionGroup, newVersion: Long, - remoteFallbackLayer: RemoteFallbackLayer, - userToken: Option[String]): Fox[Unit] = + remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[Unit] = for { actionsWithTimestamp <- Fox.successful(updateActionGroup.actions.map(_.addTimestamp(updateActionGroup.timestamp))) - _ <- dryApplyUpdates(editableMappingId, newVersion, actionsWithTimestamp, remoteFallbackLayer, userToken) ?~> "editableMapping.dryUpdate.failed" + _ <- dryApplyUpdates(editableMappingId, newVersion, actionsWithTimestamp, remoteFallbackLayer) ?~> "editableMapping.dryUpdate.failed" _ <- tracingDataStore.editableMappingUpdates.put(editableMappingId, newVersion, actionsWithTimestamp) } yield () private def dryApplyUpdates(editableMappingId: String, newVersion: Long, updates: List[UpdateAction], - remoteFallbackLayer: RemoteFallbackLayer, - userToken: Option[String]): Fox[Unit] = + remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[Unit] = for { - (previousInfo, previousVersion) <- getInfoAndActualVersion(editableMappingId, - None, - remoteFallbackLayer, - userToken) + (previousInfo, previousVersion) <- getInfoAndActualVersion(editableMappingId, None, remoteFallbackLayer) updater = new EditableMappingUpdater( editableMappingId, previousInfo.baseMappingName, previousVersion, newVersion, remoteFallbackLayer, - userToken, + tc, remoteDatastoreClient, this, tracingDataStore, @@ -269,10 +267,8 @@ class EditableMappingService @Inject()( updated <- updater.applyUpdatesAndSave(previousInfo, updates, dry = true) ?~> "editableMapping.update.failed" } yield () - def applyPendingUpdates(editableMappingId: String, - desiredVersion: Long, - remoteFallbackLayer: RemoteFallbackLayer, - userToken: Option[String]): Fox[EditableMappingInfo] = + def applyPendingUpdates(editableMappingId: String, desiredVersion: Long, remoteFallbackLayer: RemoteFallbackLayer)( + implicit tc: TokenContext): Fox[EditableMappingInfo] = for { closestMaterializedWithVersion <- getClosestMaterialized(editableMappingId, desiredVersion) updatedEditableMappingInfo: EditableMappingInfo <- if (desiredVersion == closestMaterializedWithVersion.version) @@ -286,7 +282,7 @@ class EditableMappingService @Inject()( closestMaterializedWithVersion.version, desiredVersion, remoteFallbackLayer, - userToken, + tc, remoteDatastoreClient, this, tracingDataStore, @@ -343,32 +339,28 @@ class EditableMappingService @Inject()( def findSegmentIdAtPositionIfNeeded(remoteFallbackLayer: RemoteFallbackLayer, positionOpt: Option[Vec3Int], segmentIdOpt: Option[Long], - mag: Vec3Int, - userToken: Option[String]): Fox[Long] = + mag: Vec3Int)(implicit tc: TokenContext): Fox[Long] = segmentIdOpt match { case Some(segmentId) => Fox.successful(segmentId) - case None => findSegmentIdAtPosition(remoteFallbackLayer, positionOpt, mag, userToken) + case None => findSegmentIdAtPosition(remoteFallbackLayer, positionOpt, mag) } private def findSegmentIdAtPosition(remoteFallbackLayer: RemoteFallbackLayer, positionOpt: Option[Vec3Int], - mag: Vec3Int, - userToken: Option[String]): Fox[Long] = + mag: Vec3Int)(implicit tc: TokenContext): Fox[Long] = for { pos <- positionOpt.toFox ?~> "segment id or position is required in editable mapping action" - voxelAsBytes: Array[Byte] <- remoteDatastoreClient.getVoxelAtPosition(userToken, remoteFallbackLayer, pos, mag) + voxelAsBytes: Array[Byte] <- remoteDatastoreClient.getVoxelAtPosition(remoteFallbackLayer, pos, mag) voxelAsLongArray: Array[Long] <- bytesToLongs(voxelAsBytes, remoteFallbackLayer.elementClass) _ <- Fox.bool2Fox(voxelAsLongArray.length == 1) ?~> s"Expected one, got ${voxelAsLongArray.length} segment id values for voxel." voxelAsLong <- voxelAsLongArray.headOption } yield voxelAsLong - def volumeData(tracing: VolumeTracing, - tracingId: String, - dataRequests: DataRequestCollection, - userToken: Option[String]): Fox[(Array[Byte], List[Int])] = + def volumeData(tracing: VolumeTracing, tracingId: String, dataRequests: DataRequestCollection)( + implicit tc: TokenContext): Fox[(Array[Byte], List[Int])] = for { editableMappingId <- tracing.mappingName.toFox - dataLayer = editableMappingLayer(editableMappingId, tracing, tracingId, userToken) + dataLayer = editableMappingLayer(editableMappingId, tracing, tracingId) requests = dataRequests.map(r => DataServiceDataRequest(null, dataLayer, r.cuboid(dataLayer), r.settings.copy(appliedAgglomerate = None))) data <- binaryDataService.handleDataRequests(requests) @@ -425,12 +417,12 @@ class EditableMappingService @Inject()( asSequence = valueProto.segmentToAgglomerate.map(pair => pair.segmentId -> pair.agglomerateId) } yield asSequence - def generateCombinedMappingForSegmentIds(segmentIds: Set[Long], - editableMapping: EditableMappingInfo, - editableMappingVersion: Long, - editableMappingId: String, - remoteFallbackLayer: RemoteFallbackLayer, - userToken: Option[String]): Fox[Map[Long, Long]] = + def generateCombinedMappingForSegmentIds( + segmentIds: Set[Long], + editableMapping: EditableMappingInfo, + editableMappingVersion: Long, + editableMappingId: String, + remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[Map[Long, Long]] = for { editableMappingForSegmentIds <- getSegmentToAgglomerateForSegmentIds(segmentIds, editableMappingId, @@ -439,25 +431,22 @@ class EditableMappingService @Inject()( segmentIdsInBaseMapping: Set[Long] = segmentIds.diff(segmentIdsInEditableMapping) baseMappingSubset <- getBaseSegmentToAgglomerate(editableMapping.baseMappingName, segmentIdsInBaseMapping, - remoteFallbackLayer, - userToken) + remoteFallbackLayer) } yield editableMappingForSegmentIds ++ baseMappingSubset def getAgglomerateSkeletonWithFallback(editableMappingId: String, remoteFallbackLayer: RemoteFallbackLayer, - agglomerateId: Long, - userToken: Option[String]): Fox[Array[Byte]] = + agglomerateId: Long)(implicit tc: TokenContext): Fox[Array[Byte]] = for { // called here to ensure updates are applied - editableMappingInfo <- getInfo(editableMappingId, version = None, remoteFallbackLayer, userToken) - agglomerateGraphBox <- getAgglomerateGraphForId(editableMappingId, agglomerateId, remoteFallbackLayer, userToken).futureBox + editableMappingInfo <- getInfo(editableMappingId, version = None, remoteFallbackLayer) + agglomerateGraphBox <- getAgglomerateGraphForId(editableMappingId, agglomerateId, remoteFallbackLayer).futureBox skeletonBytes <- agglomerateGraphBox match { case Full(agglomerateGraph) => Fox.successful( agglomerateGraphToSkeleton(editableMappingId, agglomerateGraph, remoteFallbackLayer, agglomerateId)) case Empty => - remoteDatastoreClient.getAgglomerateSkeleton(userToken, - remoteFallbackLayer, + remoteDatastoreClient.getAgglomerateSkeleton(remoteFallbackLayer, editableMappingInfo.baseMappingName, agglomerateId) case f: Failure => f.toFox @@ -499,16 +488,13 @@ class EditableMappingService @Inject()( skeleton.toByteArray } - def getBaseSegmentToAgglomerate(mappingName: String, - segmentIds: Set[Long], - remoteFallbackLayer: RemoteFallbackLayer, - userToken: Option[String]): Fox[Map[Long, Long]] = { + def getBaseSegmentToAgglomerate(mappingName: String, segmentIds: Set[Long], remoteFallbackLayer: RemoteFallbackLayer)( + implicit tc: TokenContext): Fox[Map[Long, Long]] = { val segmentIdsOrdered = segmentIds.toList for { agglomerateIdsOrdered <- remoteDatastoreClient.getAgglomerateIdsForSegmentIds(remoteFallbackLayer, mappingName, - segmentIdsOrdered, - userToken) + segmentIdsOrdered) } yield segmentIdsOrdered.zip(agglomerateIdsOrdered).toMap } @@ -545,29 +531,25 @@ class EditableMappingService @Inject()( bytes = UnsignedIntegerArray.toByteArray(unsignedIntArray, elementClass) } yield bytes - private def editableMappingLayer(mappingName: String, - tracing: VolumeTracing, - tracingId: String, - userToken: Option[String]): EditableMappingLayer = + private def editableMappingLayer(mappingName: String, tracing: VolumeTracing, tracingId: String)( + implicit tc: TokenContext): EditableMappingLayer = EditableMappingLayer( mappingName, tracing.boundingBox, resolutions = tracing.resolutions.map(vec3IntFromProto).toList, largestSegmentId = Some(0L), elementClass = tracing.elementClass, - userToken, + tc, tracing = tracing, tracingId = tracingId, editableMappingService = this ) - def createAdHocMesh(tracing: VolumeTracing, - tracingId: String, - request: WebknossosAdHocMeshRequest, - userToken: Option[String]): Fox[(Array[Float], List[Int])] = + def createAdHocMesh(tracing: VolumeTracing, tracingId: String, request: WebknossosAdHocMeshRequest)( + implicit tc: TokenContext): Fox[(Array[Float], List[Int])] = for { mappingName <- tracing.mappingName.toFox - segmentationLayer = editableMappingLayer(mappingName, tracing, tracingId, userToken) + segmentationLayer = editableMappingLayer(mappingName, tracing, tracingId) adHocMeshRequest = AdHocMeshRequest( dataSource = None, dataLayer = segmentationLayer, @@ -581,14 +563,14 @@ class EditableMappingService @Inject()( result <- adHocMeshService.requestAdHocMeshViaActor(adHocMeshRequest) } yield result - def getAgglomerateGraphForId(mappingId: String, - agglomerateId: Long, - remoteFallbackLayer: RemoteFallbackLayer, - userToken: Option[String], - requestedVersion: Option[Long] = None): Fox[AgglomerateGraph] = + def getAgglomerateGraphForId( + mappingId: String, + agglomerateId: Long, + remoteFallbackLayer: RemoteFallbackLayer, + requestedVersion: Option[Long] = None)(implicit tc: TokenContext): Fox[AgglomerateGraph] = for { // called here to ensure updates are applied - (_, version) <- getInfoAndActualVersion(mappingId, requestedVersion, remoteFallbackLayer, userToken) + (_, version) <- getInfoAndActualVersion(mappingId, requestedVersion, remoteFallbackLayer) agglomerateGraph <- agglomerateToGraphCache.getOrLoad( (mappingId, agglomerateId, version), _ => @@ -601,41 +583,32 @@ class EditableMappingService @Inject()( ) } yield agglomerateGraph - def getAgglomerateGraphForIdWithFallback(mapping: EditableMappingInfo, - editableMappingId: String, - version: Option[Long], - agglomerateId: Long, - remoteFallbackLayer: RemoteFallbackLayer, - userToken: Option[String]): Fox[AgglomerateGraph] = - for { - agglomerateGraphBox <- getAgglomerateGraphForId(editableMappingId, - agglomerateId, - remoteFallbackLayer, - userToken, - version).futureBox + def getAgglomerateGraphForIdWithFallback( + mapping: EditableMappingInfo, + editableMappingId: String, + version: Option[Long], + agglomerateId: Long, + remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[AgglomerateGraph] = + for { + agglomerateGraphBox <- getAgglomerateGraphForId(editableMappingId, agglomerateId, remoteFallbackLayer, version).futureBox agglomerateGraph <- agglomerateGraphBox match { case Full(agglomerateGraph) => Fox.successful(agglomerateGraph) case Empty => - remoteDatastoreClient.getAgglomerateGraph(remoteFallbackLayer, - mapping.baseMappingName, - agglomerateId, - userToken) + remoteDatastoreClient.getAgglomerateGraph(remoteFallbackLayer, mapping.baseMappingName, agglomerateId) case f: Failure => f.toFox } } yield agglomerateGraph - def agglomerateGraphMinCut(parameters: MinCutParameters, - remoteFallbackLayer: RemoteFallbackLayer, - userToken: Option[String]): Fox[List[EdgeWithPositions]] = + def agglomerateGraphMinCut(parameters: MinCutParameters, remoteFallbackLayer: RemoteFallbackLayer)( + implicit tc: TokenContext): Fox[List[EdgeWithPositions]] = for { // called here to ensure updates are applied - mapping <- getInfo(parameters.editableMappingId, version = None, remoteFallbackLayer, userToken) + mapping <- getInfo(parameters.editableMappingId, version = None, remoteFallbackLayer) agglomerateGraph <- getAgglomerateGraphForIdWithFallback(mapping, parameters.editableMappingId, None, parameters.agglomerateId, - remoteFallbackLayer, - userToken) + remoteFallbackLayer) edgesToCut <- minCut(agglomerateGraph, parameters.segmentId1, parameters.segmentId2) ?~> "Could not calculate min-cut on agglomerate graph." edgesWithPositions = annotateEdgesWithPositions(edgesToCut, agglomerateGraph) } yield edgesWithPositions @@ -692,18 +665,16 @@ class EditableMappingService @Inject()( ) } - def agglomerateGraphNeighbors(parameters: NeighborsParameters, - remoteFallbackLayer: RemoteFallbackLayer, - userToken: Option[String]): Fox[(Long, Seq[NodeWithPosition])] = + def agglomerateGraphNeighbors(parameters: NeighborsParameters, remoteFallbackLayer: RemoteFallbackLayer)( + implicit tc: TokenContext): Fox[(Long, Seq[NodeWithPosition])] = for { // called here to ensure updates are applied - mapping <- getInfo(parameters.editableMappingId, version = None, remoteFallbackLayer, userToken) + mapping <- getInfo(parameters.editableMappingId, version = None, remoteFallbackLayer) agglomerateGraph <- getAgglomerateGraphForIdWithFallback(mapping, parameters.editableMappingId, None, parameters.agglomerateId, - remoteFallbackLayer, - userToken) + remoteFallbackLayer) neighborNodes = neighbors(agglomerateGraph, parameters.segmentId) nodesWithPositions = annotateNodesWithPositions(neighborNodes, agglomerateGraph) } yield (parameters.segmentId, nodesWithPositions) @@ -718,29 +689,24 @@ class EditableMappingService @Inject()( neighborNodes } - def merge(editableMappingIds: List[String], - remoteFallbackLayer: RemoteFallbackLayer, - userToken: Option[String]): Fox[String] = + def merge(editableMappingIds: List[String], remoteFallbackLayer: RemoteFallbackLayer)( + implicit tc: TokenContext): Fox[String] = for { firstMappingId <- editableMappingIds.headOption.toFox before = Instant.now - newMappingId <- duplicate(Some(firstMappingId), version = None, Some(remoteFallbackLayer), userToken) + newMappingId <- duplicate(Some(firstMappingId), version = None, Some(remoteFallbackLayer)) _ <- Fox.serialCombined(editableMappingIds.tail)(editableMappingId => - mergeInto(newMappingId, editableMappingId, remoteFallbackLayer, userToken)) + mergeInto(newMappingId, editableMappingId, remoteFallbackLayer)) _ = logger.info(s"Merging ${editableMappingIds.length} editable mappings took ${Instant.since(before)}") } yield newMappingId // read as: merge source into target (mutate target) private def mergeInto(targetEditableMappingId: String, sourceEditableMappingId: String, - remoteFallbackLayer: RemoteFallbackLayer, - userToken: Option[String]): Fox[Unit] = + remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[Unit] = for { targetNewestVersion <- getClosestMaterializableVersionOrZero(targetEditableMappingId, None) - sourceNewestMaterializedWithVersion <- getInfoAndActualVersion(sourceEditableMappingId, - None, - remoteFallbackLayer, - userToken) + sourceNewestMaterializedWithVersion <- getInfoAndActualVersion(sourceEditableMappingId, None, remoteFallbackLayer) sourceNewestVersion = sourceNewestMaterializedWithVersion._2 updateActionsWithVersions <- getUpdateActionsWithVersions(sourceEditableMappingId, sourceNewestVersion, 0L) updateActionsToApply = updateActionsWithVersions.map(_._2).reverse.flatten @@ -750,7 +716,7 @@ class EditableMappingService @Inject()( targetNewestVersion, targetNewestVersion + sourceNewestVersion, remoteFallbackLayer, - userToken, + tc, remoteDatastoreClient, this, tracingDataStore, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index 9f1b7c1a0d8..c2a281f39f9 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -1,5 +1,6 @@ package com.scalableminds.webknossos.tracingstore.tracings.editablemapping +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.geometry.Vec3Int import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.datastore.AgglomerateGraph.{AgglomerateEdge, AgglomerateGraph} @@ -36,7 +37,7 @@ class EditableMappingUpdater( oldVersion: Long, newVersion: Long, remoteFallbackLayer: RemoteFallbackLayer, - userToken: Option[String], + tokenContext: TokenContext, remoteDatastoreClient: TSRemoteDatastoreClient, editableMappingService: EditableMappingService, tracingDataStore: TracingDataStore, @@ -127,13 +128,11 @@ class EditableMappingUpdater( segmentId1 <- editableMappingService.findSegmentIdAtPositionIfNeeded(remoteFallbackLayer, update.segmentPosition1, update.segmentId1, - update.mag, - userToken) + update.mag)(tokenContext) segmentId2 <- editableMappingService.findSegmentIdAtPositionIfNeeded(remoteFallbackLayer, update.segmentPosition2, update.segmentId2, - update.mag, - userToken) + update.mag)(tokenContext) agglomerateId <- agglomerateIdForSplitAction(update, segmentId1) agglomerateGraph <- agglomerateGraphForIdWithFallback(editableMappingInfo, agglomerateId) _ = if (segmentId1 == 0) @@ -196,7 +195,7 @@ class EditableMappingUpdater( case Some(agglomerateId) => Fox.successful(agglomerateId) case None => editableMappingService - .getBaseSegmentToAgglomerate(baseMappingName, Set(segmentId), remoteFallbackLayer, userToken) + .getBaseSegmentToAgglomerate(baseMappingName, Set(segmentId), remoteFallbackLayer)(tokenContext) .flatMap(baseSegmentToAgglomerate => baseSegmentToAgglomerate.get(segmentId)) } } yield agglomerateId @@ -241,7 +240,7 @@ class EditableMappingUpdater( Some(oldVersion), agglomerateId, remoteFallbackLayer, - userToken) + )(tokenContext) } } @@ -336,8 +335,7 @@ class EditableMappingUpdater( private def largestAgglomerateId(mapping: EditableMappingInfo): Fox[Long] = for { largestBaseAgglomerateId <- remoteDatastoreClient.getLargestAgglomerateId(remoteFallbackLayer, - mapping.baseMappingName, - userToken) + mapping.baseMappingName)(tokenContext) } yield math.max(mapping.largestAgglomerateId, largestBaseAgglomerateId) private def applyMergeAction(mapping: EditableMappingInfo, update: MergeAgglomerateUpdateAction)( @@ -346,13 +344,11 @@ class EditableMappingUpdater( segmentId1 <- editableMappingService.findSegmentIdAtPositionIfNeeded(remoteFallbackLayer, update.segmentPosition1, update.segmentId1, - update.mag, - userToken) + update.mag)(tokenContext) segmentId2 <- editableMappingService.findSegmentIdAtPositionIfNeeded(remoteFallbackLayer, update.segmentPosition2, update.segmentId2, - update.mag, - userToken) + update.mag)(tokenContext) _ = if (segmentId1 == 0) logger.warn( s"Merge action for editable mapping $editableMappingId: Looking up segment id at position ${update.segmentPosition1} in mag ${update.mag} returned invalid value zero. Merging outside of dataset?") @@ -420,8 +416,7 @@ class EditableMappingUpdater( _ <- bool2Fox(revertAction.sourceVersion <= oldVersion) ?~> "trying to revert editable mapping to a version not yet present in the database" oldInfo <- editableMappingService.getInfo(editableMappingId, Some(revertAction.sourceVersion), - remoteFallbackLayer, - userToken) + remoteFallbackLayer)(tokenContext) _ = segmentToAgglomerateBuffer.clear() _ = agglomerateToGraphBuffer.clear() segmentToAgglomerateChunkNewestStream = new VersionedSegmentToAgglomerateChunkIterator( @@ -453,8 +448,7 @@ class EditableMappingUpdater( .getAgglomerateGraphForId(editableMappingId, agglomerateId, remoteFallbackLayer, - userToken, - Some(revertAction.sourceVersion)) + Some(revertAction.sourceVersion))(tokenContext) .futureBox .map { case Full(graphData) => agglomerateToGraphBuffer.put(graphKey, (graphData, false)) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala index e07a99558da..aaff678f9c3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala @@ -1,6 +1,7 @@ package com.scalableminds.webknossos.tracingstore.tracings.skeleton import com.google.inject.Inject +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int} import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing @@ -8,7 +9,7 @@ import com.scalableminds.webknossos.datastore.geometry.NamedBoundingBoxProto import com.scalableminds.webknossos.datastore.helpers.{ProtoGeometryImplicits, SkeletonTracingDefaults} import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis import com.scalableminds.webknossos.tracingstore.TracingStoreRedisStore -import com.scalableminds.webknossos.tracingstore.annotation.{AnnotationWithTracings, TSAnnotationService} +import com.scalableminds.webknossos.tracingstore.annotation.TSAnnotationService import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.webknossos.tracingstore.tracings.volume.MergedVolumeStats import net.liftweb.common.{Box, Full} @@ -44,13 +45,12 @@ class SkeletonTracingService @Inject()( tracingId: String, version: Option[Long] = None, useCache: Boolean = true, - applyUpdates: Boolean = false, - userToken: Option[String]): Fox[SkeletonTracing] = + applyUpdates: Boolean = false)(implicit tc: TokenContext): Fox[SkeletonTracing] = if (tracingId == TracingIds.dummyTracingId) Fox.successful(dummyTracing) else { for { - annotation <- annotationService.getWithTracings(annotationId, version, List(tracingId), List.empty, userToken) // TODO is applyUpdates still needed? + annotation <- annotationService.getWithTracings(annotationId, version, List(tracingId), List.empty) // TODO is applyUpdates still needed? tracing <- annotation.getSkeleton(tracingId) } yield tracing } @@ -128,12 +128,11 @@ class SkeletonTracingService @Inject()( tracings: Seq[SkeletonTracing], newId: String, newVersion: Long, - toCache: Boolean, - userToken: Option[String])(implicit mp: MessagesProvider): Fox[MergedVolumeStats] = + toCache: Boolean)(implicit mp: MessagesProvider, tc: TokenContext): Fox[MergedVolumeStats] = Fox.successful(MergedVolumeStats.empty()) def dummyTracing: SkeletonTracing = SkeletonTracingDefaults.createInstance - def mergeEditableMappings(tracingsWithIds: List[(SkeletonTracing, String)], userToken: Option[String]): Fox[String] = + def mergeEditableMappings(tracingsWithIds: List[(SkeletonTracing, String)])(implicit tc: TokenContext): Fox[String] = Fox.empty } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala index 726de0f8af9..381f3420b5a 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala @@ -1,5 +1,6 @@ package com.scalableminds.webknossos.tracingstore.tracings.volume +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.geometry.Vec3Int import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox @@ -33,20 +34,19 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, with FullMeshHelper with LazyLogging { - def loadFor(token: Option[String], annotationId: String, tracingId: String, fullMeshRequest: FullMeshRequest)( - implicit ec: ExecutionContext): Fox[Array[Byte]] = + def loadFor(annotationId: String, tracingId: String, fullMeshRequest: FullMeshRequest)( + implicit ec: ExecutionContext, + tc: TokenContext): Fox[Array[Byte]] = for { - tracing <- volumeTracingService.find(annotationId, tracingId, userToken = token) ?~> "tracing.notFound" + tracing <- volumeTracingService.find(annotationId, tracingId) ?~> "tracing.notFound" data <- if (fullMeshRequest.meshFileName.isDefined) - loadFullMeshFromMeshfile(token, tracing, tracingId, fullMeshRequest) - else loadFullMeshFromAdHoc(token, tracing, annotationId, tracingId, fullMeshRequest) + loadFullMeshFromMeshfile(tracing, tracingId, fullMeshRequest) + else loadFullMeshFromAdHoc(tracing, annotationId, tracingId, fullMeshRequest) } yield data - private def loadFullMeshFromMeshfile( - token: Option[String], - tracing: VolumeTracing, - tracingId: String, - fullMeshRequest: FullMeshRequest)(implicit ec: ExecutionContext): Fox[Array[Byte]] = + private def loadFullMeshFromMeshfile(tracing: VolumeTracing, tracingId: String, fullMeshRequest: FullMeshRequest)( + implicit ec: ExecutionContext, + tc: TokenContext): Fox[Array[Byte]] = for { remoteFallbackLayer <- remoteFallbackLayerFromVolumeTracing(tracing, tracingId) baseMappingName <- volumeTracingService.baseMappingName(tracing) @@ -55,24 +55,23 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, editableMappingTracingId = Some(tracingId), mappingType = Some("HDF5")) else fullMeshRequest - array <- remoteDatastoreClient.loadFullMeshStl(token, remoteFallbackLayer, fullMeshRequestAdapted) + array <- remoteDatastoreClient.loadFullMeshStl(remoteFallbackLayer, fullMeshRequestAdapted) } yield array - private def loadFullMeshFromAdHoc(token: Option[String], - tracing: VolumeTracing, - annotationId: String, - tracingId: String, - fullMeshRequest: FullMeshRequest)(implicit ec: ExecutionContext): Fox[Array[Byte]] = + private def loadFullMeshFromAdHoc( + tracing: VolumeTracing, + annotationId: String, + tracingId: String, + fullMeshRequest: FullMeshRequest)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Array[Byte]] = for { mag <- fullMeshRequest.mag.toFox ?~> "mag.neededForAdHoc" _ <- bool2Fox(tracing.resolutions.contains(vec3IntToProto(mag))) ?~> "mag.notPresentInTracing" before = Instant.now - voxelSize <- remoteDatastoreClient.voxelSizeForTracingWithCache(tracingId, token) ?~> "voxelSize.failedToFetch" + voxelSize <- remoteDatastoreClient.voxelSizeForTracingWithCache(tracingId) ?~> "voxelSize.failedToFetch" verticesForChunks <- if (tracing.hasSegmentIndex.getOrElse(false)) - getAllAdHocChunksWithSegmentIndex(token, annotationId, tracing, tracingId, mag, voxelSize, fullMeshRequest) + getAllAdHocChunksWithSegmentIndex(annotationId, tracing, tracingId, mag, voxelSize, fullMeshRequest) else getAllAdHocChunksWithNeighborLogic( - token, tracing, annotationId, tracingId, @@ -88,15 +87,14 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, } yield array private def getAllAdHocChunksWithSegmentIndex( - token: Option[String], annotationId: String, tracing: VolumeTracing, tracingId: String, mag: Vec3Int, voxelSize: VoxelSize, - fullMeshRequest: FullMeshRequest)(implicit ec: ExecutionContext): Fox[List[Array[Float]]] = + fullMeshRequest: FullMeshRequest)(implicit ec: ExecutionContext, tc: TokenContext): Fox[List[Array[Float]]] = for { - fallbackLayer <- volumeTracingService.getFallbackLayer(annotationId, tracingId, userToken = token) + fallbackLayer <- volumeTracingService.getFallbackLayer(annotationId, tracingId) mappingName <- volumeTracingService.baseMappingName(tracing) bucketPositionsRaw: ListOfVec3IntProto <- volumeSegmentIndexService .getSegmentToBucketIndexWithEmptyFallbackWithoutBuffer( @@ -108,8 +106,7 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, mappingName = mappingName, editableMappingTracingId = volumeTracingService.editableMappingTracingId(tracing, tracingId), fullMeshRequest.additionalCoordinates, - AdditionalAxis.fromProtosAsOpt(tracing.additionalAxes), - token + AdditionalAxis.fromProtosAsOpt(tracing.additionalAxes) ) bucketPositions = bucketPositionsRaw.values .map(vec3IntFromProto) @@ -129,13 +126,12 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, fullMeshRequest.additionalCoordinates, findNeighbors = false ) - loadMeshChunkFromAdHoc(token, tracing, adHocMeshRequest, annotationId, tracingId) + loadMeshChunkFromAdHoc(tracing, adHocMeshRequest, annotationId, tracingId) } allVertices = vertexChunksWithNeighbors.map(_._1) } yield allVertices - private def getAllAdHocChunksWithNeighborLogic(token: Option[String], - tracing: VolumeTracing, + private def getAllAdHocChunksWithNeighborLogic(tracing: VolumeTracing, annotationId: String, tracingId: String, mag: Vec3Int, @@ -145,7 +141,8 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, chunkSize: Vec3Int, visited: collection.mutable.Set[VoxelPosition] = collection.mutable.Set[VoxelPosition]())( - implicit ec: ExecutionContext): Fox[List[Array[Float]]] = + implicit ec: ExecutionContext, + tc: TokenContext): Fox[List[Array[Float]]] = for { topLeft <- topLeftOpt.toFox ?~> "seedPosition.neededForAdHoc" adHocMeshRequest = WebknossosAdHocMeshRequest( @@ -159,16 +156,11 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, fullMeshRequest.additionalCoordinates ) _ = visited += topLeft - (vertices: Array[Float], neighbors) <- loadMeshChunkFromAdHoc(token, - tracing, - adHocMeshRequest, - annotationId, - tracingId) + (vertices: Array[Float], neighbors) <- loadMeshChunkFromAdHoc(tracing, adHocMeshRequest, annotationId, tracingId) nextPositions: List[VoxelPosition] = generateNextTopLeftsFromNeighbors(topLeft, neighbors, chunkSize, visited) _ = visited ++= nextPositions neighborVerticesNested <- Fox.serialCombined(nextPositions) { position: VoxelPosition => - getAllAdHocChunksWithNeighborLogic(token, - tracing, + getAllAdHocChunksWithNeighborLogic(tracing, annotationId, tracingId, mag, @@ -181,12 +173,11 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, allVertices: List[Array[Float]] = vertices +: neighborVerticesNested.flatten } yield allVertices - private def loadMeshChunkFromAdHoc(token: Option[String], - tracing: VolumeTracing, + private def loadMeshChunkFromAdHoc(tracing: VolumeTracing, adHocMeshRequest: WebknossosAdHocMeshRequest, annotationId: String, - tracingId: String): Fox[(Array[Float], List[Int])] = + tracingId: String)(implicit tc: TokenContext): Fox[(Array[Float], List[Int])] = if (tracing.getHasEditableMapping) - editableMappingService.createAdHocMesh(tracing, tracingId, adHocMeshRequest, token) - else volumeTracingService.createAdHocMesh(annotationId, tracingId, adHocMeshRequest, token) + editableMappingService.createAdHocMesh(tracing, tracingId, adHocMeshRequest) + else volumeTracingService.createAdHocMesh(annotationId, tracingId, adHocMeshRequest) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentIndexBuffer.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentIndexBuffer.scala index 755cc665464..9d0d35cf2e3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentIndexBuffer.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentIndexBuffer.scala @@ -1,5 +1,6 @@ package com.scalableminds.webknossos.tracingstore.tracings.volume +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.geometry.Vec3Int import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.geometry.ListOfVec3IntProto @@ -33,7 +34,7 @@ class VolumeSegmentIndexBuffer(tracingId: String, remoteDatastoreClient: TSRemoteDatastoreClient, fallbackLayer: Option[RemoteFallbackLayer], additionalAxes: Option[Seq[AdditionalAxis]], - userToken: Option[String]) + tc: TokenContext) extends KeyValueStoreImplicits with SegmentIndexKeyHelper with ProtoGeometryImplicits @@ -86,12 +87,7 @@ class VolumeSegmentIndexBuffer(tracingId: String, .fillEmpty(ListOfVec3IntProto.of(Seq())) data <- fallbackLayer match { case Some(layer) if fossilDbData.length == 0 => - remoteDatastoreClient.querySegmentIndex(layer, - segmentId, - mag, - mappingName, - editableMappingTracingId, - userToken) + remoteDatastoreClient.querySegmentIndex(layer, segmentId, mag, mappingName, editableMappingTracingId)(tc) case _ => Fox.successful(fossilDbData.values.map(vec3IntFromProto)) } } yield ListOfVec3IntProto(data.map(vec3IntToProto)) @@ -168,13 +164,8 @@ class VolumeSegmentIndexBuffer(tracingId: String, fileBucketPositions <- fallbackLayer match { case Some(layer) => for { - fileBucketPositionsOpt <- Fox.runIf(missesSoFar.nonEmpty)( - remoteDatastoreClient.querySegmentIndexForMultipleSegments(layer, - missesSoFar, - mag, - mappingName, - editableMappingTracingId, - userToken)) + fileBucketPositionsOpt <- Fox.runIf(missesSoFar.nonEmpty)(remoteDatastoreClient + .querySegmentIndexForMultipleSegments(layer, missesSoFar, mag, mappingName, editableMappingTracingId)(tc)) fileBucketPositions = fileBucketPositionsOpt.getOrElse(Seq()) _ = fileBucketPositions.map { case (segmentId, positions) => diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentIndexService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentIndexService.scala index 150c4938bb6..a88c863f7ff 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentIndexService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentIndexService.scala @@ -1,6 +1,7 @@ package com.scalableminds.webknossos.tracingstore.tracings.volume import com.google.inject.Inject +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.geometry.Vec3Int import com.scalableminds.util.tools.Fox import com.scalableminds.util.tools.Fox.box2Fox @@ -25,11 +26,11 @@ import net.liftweb.common.Box.tryo import scala.concurrent.ExecutionContext object VolumeSegmentIndexService { - def canHaveSegmentIndex(remoteDatastoreClient: TSRemoteDatastoreClient, - fallbackLayer: Option[RemoteFallbackLayer], - userToken: Option[String])(implicit ec: ExecutionContext): Fox[Boolean] = + def canHaveSegmentIndex(remoteDatastoreClient: TSRemoteDatastoreClient, fallbackLayer: Option[RemoteFallbackLayer])( + implicit ec: ExecutionContext, + tc: TokenContext): Fox[Boolean] = fallbackLayer match { - case Some(layer) => remoteDatastoreClient.hasSegmentIndexFile(layer, userToken) + case Some(layer) => remoteDatastoreClient.hasSegmentIndexFile(layer) case None => Fox.successful(true) } } @@ -158,17 +159,17 @@ class VolumeSegmentIndexService @Inject()(val tracingDataStore: TracingDataStore bucketList <- addEmptyFallback(bucketListBox) } yield bucketList - def getSegmentToBucketIndexWithEmptyFallbackWithoutBuffer( - fallbackLayer: Option[RemoteFallbackLayer], - tracingId: String, - segmentId: Long, - mag: Vec3Int, - version: Option[Long] = None, - mappingName: Option[String], - editableMappingTracingId: Option[String], - additionalCoordinates: Option[Seq[AdditionalCoordinate]], - additionalAxes: Option[Seq[AdditionalAxis]], - userToken: Option[String])(implicit ec: ExecutionContext): Fox[ListOfVec3IntProto] = + def getSegmentToBucketIndexWithEmptyFallbackWithoutBuffer(fallbackLayer: Option[RemoteFallbackLayer], + tracingId: String, + segmentId: Long, + mag: Vec3Int, + version: Option[Long] = None, + mappingName: Option[String], + editableMappingTracingId: Option[String], + additionalCoordinates: Option[Seq[AdditionalCoordinate]], + additionalAxes: Option[Seq[AdditionalAxis]])( + implicit ec: ExecutionContext, + tc: TokenContext): Fox[ListOfVec3IntProto] = for { bucketListBox <- getSegmentToBucketIndex(fallbackLayer, tracingId, @@ -178,8 +179,7 @@ class VolumeSegmentIndexService @Inject()(val tracingDataStore: TracingDataStore mappingName, editableMappingTracingId, additionalCoordinates, - additionalAxes, - userToken).futureBox + additionalAxes).futureBox bucketList <- addEmptyFallback(bucketListBox) } yield bucketList @@ -191,17 +191,17 @@ class VolumeSegmentIndexService @Inject()(val tracingDataStore: TracingDataStore case Empty => Fox.successful(ListOfVec3IntProto(Seq.empty)) } - private def getSegmentToBucketIndex( - fallbackLayerOpt: Option[RemoteFallbackLayer], - tracingId: String, - segmentId: Long, - mag: Vec3Int, - version: Option[Long], - mappingName: Option[String], - editableMappingTracingId: Option[String], - additionalCoordinates: Option[Seq[AdditionalCoordinate]], - additionalAxes: Option[Seq[AdditionalAxis]], - userToken: Option[String])(implicit ec: ExecutionContext): Fox[ListOfVec3IntProto] = + private def getSegmentToBucketIndex(fallbackLayerOpt: Option[RemoteFallbackLayer], + tracingId: String, + segmentId: Long, + mag: Vec3Int, + version: Option[Long], + mappingName: Option[String], + editableMappingTracingId: Option[String], + additionalCoordinates: Option[Seq[AdditionalCoordinate]], + additionalAxes: Option[Seq[AdditionalAxis]])( + implicit ec: ExecutionContext, + tc: TokenContext): Fox[ListOfVec3IntProto] = for { fromMutableIndex <- getSegmentToBucketIndexFromFossilDB(tracingId, segmentId, @@ -211,12 +211,7 @@ class VolumeSegmentIndexService @Inject()(val tracingDataStore: TracingDataStore additionalAxes).fillEmpty(ListOfVec3IntProto.of(Seq())) fromFileIndex <- fallbackLayerOpt match { // isEmpty is not the same as length == 0 here :( case Some(fallbackLayer) if fromMutableIndex.length == 0 => - getSegmentToBucketIndexFromFile(fallbackLayer, - segmentId, - mag, - mappingName, - editableMappingTracingId, - userToken) // additional coordinates not supported, see #7556 + getSegmentToBucketIndexFromFile(fallbackLayer, segmentId, mag, mappingName, editableMappingTracingId) // additional coordinates not supported, see #7556 case _ => Fox.successful(Seq.empty) } combined = fromMutableIndex.values.map(vec3IntFromProto) ++ fromFileIndex @@ -237,8 +232,7 @@ class VolumeSegmentIndexService @Inject()(val tracingDataStore: TracingDataStore segmentId: Long, mag: Vec3Int, mappingName: Option[String], - editableMappingTracingId: Option[String], - userToken: Option[String]) = - remoteDatastoreClient.querySegmentIndex(layer, segmentId, mag, mappingName, editableMappingTracingId, userToken) + editableMappingTracingId: Option[String])(implicit tc: TokenContext) = + remoteDatastoreClient.querySegmentIndex(layer, segmentId, mag, mappingName, editableMappingTracingId) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentStatisticsService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentStatisticsService.scala index a185e2b3735..6ad05f26680 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentStatisticsService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentStatisticsService.scala @@ -1,5 +1,6 @@ package com.scalableminds.webknossos.tracingstore.tracings.volume +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.geometry.{BoundingBox, Vec3Int} import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing @@ -26,14 +27,14 @@ class VolumeSegmentStatisticsService @Inject()(volumeTracingService: VolumeTraci segmentId: Long, mag: Vec3Int, mappingName: Option[String], - additionalCoordinates: Option[Seq[AdditionalCoordinate]], - userToken: Option[String])(implicit ec: ExecutionContext): Fox[Long] = + additionalCoordinates: Option[Seq[AdditionalCoordinate]])(implicit ec: ExecutionContext, + tc: TokenContext): Fox[Long] = calculateSegmentVolume( segmentId, mag, additionalCoordinates, - getBucketPositions(annotationId, tracingId, mappingName, additionalCoordinates, userToken), - getTypedDataForBucketPosition(annotationId, tracingId, userToken) + getBucketPositions(annotationId, tracingId, mappingName, additionalCoordinates), + getTypedDataForBucketPosition(annotationId, tracingId) ) def getSegmentBoundingBox(annotationId: String, @@ -41,42 +42,38 @@ class VolumeSegmentStatisticsService @Inject()(volumeTracingService: VolumeTraci segmentId: Long, mag: Vec3Int, mappingName: Option[String], - additionalCoordinates: Option[Seq[AdditionalCoordinate]], - userToken: Option[String])(implicit ec: ExecutionContext): Fox[BoundingBox] = + additionalCoordinates: Option[Seq[AdditionalCoordinate]])( + implicit ec: ExecutionContext, + tc: TokenContext): Fox[BoundingBox] = calculateSegmentBoundingBox( segmentId, mag, additionalCoordinates, - getBucketPositions(annotationId, tracingId, mappingName, additionalCoordinates, userToken), - getTypedDataForBucketPosition(annotationId, tracingId, userToken) + getBucketPositions(annotationId, tracingId, mappingName, additionalCoordinates), + getTypedDataForBucketPosition(annotationId, tracingId) ) - private def getTypedDataForBucketPosition(annotationId: String, tracingId: String, userToken: Option[String])( + private def getTypedDataForBucketPosition(annotationId: String, tracingId: String)( bucketPosition: Vec3Int, mag: Vec3Int, - additionalCoordinates: Option[Seq[AdditionalCoordinate]]) = + additionalCoordinates: Option[Seq[AdditionalCoordinate]])(implicit tc: TokenContext) = for { - tracing <- volumeTracingService.find(annotationId, tracingId, userToken = userToken) ?~> "tracing.notFound" - bucketData <- getVolumeDataForPositions(tracing, - tracingId, - mag, - Seq(bucketPosition), - additionalCoordinates, - userToken) + tracing <- volumeTracingService.find(annotationId, tracingId) ?~> "tracing.notFound" + bucketData <- getVolumeDataForPositions(tracing, tracingId, mag, Seq(bucketPosition), additionalCoordinates) dataTyped: Array[UnsignedInteger] = UnsignedIntegerArray.fromByteArray( bucketData, elementClassFromProto(tracing.elementClass)) } yield dataTyped - private def getBucketPositions( - annotationId: String, - tracingId: String, - mappingName: Option[String], - additionalCoordinates: Option[Seq[AdditionalCoordinate]], - userToken: Option[String])(segmentId: Long, mag: Vec3Int)(implicit ec: ExecutionContext) = + private def getBucketPositions(annotationId: String, + tracingId: String, + mappingName: Option[String], + additionalCoordinates: Option[Seq[AdditionalCoordinate]])( + segmentId: Long, + mag: Vec3Int)(implicit ec: ExecutionContext, tc: TokenContext) = for { - fallbackLayer <- volumeTracingService.getFallbackLayer(annotationId, tracingId, userToken) - tracing <- volumeTracingService.find(annotationId, tracingId, userToken = userToken) ?~> "tracing.notFound" + fallbackLayer <- volumeTracingService.getFallbackLayer(annotationId, tracingId) + tracing <- volumeTracingService.find(annotationId, tracingId) ?~> "tracing.notFound" additionalAxes = AdditionalAxis.fromProtosAsOpt(tracing.additionalAxes) allBucketPositions: ListOfVec3IntProto <- volumeSegmentIndexService .getSegmentToBucketIndexWithEmptyFallbackWithoutBuffer( @@ -88,8 +85,7 @@ class VolumeSegmentStatisticsService @Inject()(volumeTracingService: VolumeTraci mappingName, editableMappingTracingId = volumeTracingService.editableMappingTracingId(tracing, tracingId), additionalCoordinates, - additionalAxes, - userToken + additionalAxes ) } yield allBucketPositions @@ -97,8 +93,7 @@ class VolumeSegmentStatisticsService @Inject()(volumeTracingService: VolumeTraci tracingId: String, mag: Vec3Int, bucketPositions: Seq[Vec3Int], - additionalCoordinates: Option[Seq[AdditionalCoordinate]], - userToken: Option[String]): Fox[Array[Byte]] = { + additionalCoordinates: Option[Seq[AdditionalCoordinate]])(implicit tc: TokenContext): Fox[Array[Byte]] = { val dataRequests = bucketPositions.map { position => WebknossosDataRequest( @@ -113,8 +108,8 @@ class VolumeSegmentStatisticsService @Inject()(volumeTracingService: VolumeTraci }.toList for { (data, _) <- if (tracing.getHasEditableMapping) - editableMappingService.volumeData(tracing, tracingId, dataRequests, userToken) - else volumeTracingService.data(tracingId, tracing, dataRequests, includeFallbackDataIfAvailable = true, userToken) + editableMappingService.volumeData(tracing, tracingId, dataRequests) + else volumeTracingService.data(tracingId, tracing, dataRequests, includeFallbackDataIfAvailable = true) } yield data } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala index 3aa03f8be6d..0f3e8d06d5c 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala @@ -216,22 +216,20 @@ trait VolumeTracingBucketHelper } } - private def loadFallbackBucket(dataLayer: VolumeTracingLayer, bucket: BucketPosition): Fox[Array[Byte]] = { + private def loadFallbackBucket(layer: VolumeTracingLayer, bucket: BucketPosition): Fox[Array[Byte]] = { val dataRequest: WebknossosDataRequest = WebknossosDataRequest( position = Vec3Int(bucket.topLeft.mag1X, bucket.topLeft.mag1Y, bucket.topLeft.mag1Z), mag = bucket.mag, - cubeSize = dataLayer.lengthOfUnderlyingCubes(bucket.mag), + cubeSize = layer.lengthOfUnderlyingCubes(bucket.mag), fourBit = None, - applyAgglomerate = dataLayer.tracing.mappingName, + applyAgglomerate = layer.tracing.mappingName, version = None, additionalCoordinates = None ) for { - remoteFallbackLayer <- dataLayer.volumeTracingService - .remoteFallbackLayerFromVolumeTracing(dataLayer.tracing, dataLayer.name) - (unmappedData, indices) <- dataLayer.volumeTracingService.getFallbackDataFromDatastore(remoteFallbackLayer, - List(dataRequest), - dataLayer.userToken) + remoteFallbackLayer <- layer.volumeTracingService.remoteFallbackLayerFromVolumeTracing(layer.tracing, layer.name) + (unmappedData, indices) <- layer.volumeTracingService + .getFallbackDataFromDatastore(remoteFallbackLayer, List(dataRequest))(ec, layer.tokenContext) unmappedDataOrEmpty <- if (indices.isEmpty) Fox.successful(unmappedData) else Fox.empty } yield unmappedDataOrEmpty } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala index 25f59bb4bda..97cf49ed688 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala @@ -1,5 +1,6 @@ package com.scalableminds.webknossos.tracingstore.tracings.volume +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.geometry.Vec3Int import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.datastore.models.{BucketPosition, UnsignedIntegerArray} @@ -77,13 +78,13 @@ trait VolumeTracingDownsampling protected def volumeSegmentIndexClient: FossilDBClient - protected def downsampleWithLayer(annotationId: String, - tracingId: String, - oldTracingId: String, - tracing: VolumeTracing, - dataLayer: VolumeTracingLayer, - tracingService: VolumeTracingService, - userToken: Option[String])(implicit ec: ExecutionContext): Fox[List[Vec3Int]] = { + protected def downsampleWithLayer( + annotationId: String, + tracingId: String, + oldTracingId: String, + tracing: VolumeTracing, + dataLayer: VolumeTracingLayer, + tracingService: VolumeTracingService)(implicit ec: ExecutionContext, tc: TokenContext): Fox[List[Vec3Int]] = { val bucketVolume = 32 * 32 * 32 for { _ <- bool2Fox(tracing.version == 0L) ?~> "Tracing has already been edited." @@ -106,15 +107,15 @@ trait VolumeTracingDownsampling dataLayer) requiredMag } - fallbackLayer <- tracingService.getFallbackLayer(annotationId, oldTracingId, userToken) // remote wk does not know the new id yet - tracing <- tracingService.find(annotationId, tracingId, userToken = userToken) ?~> "tracing.notFound" + fallbackLayer <- tracingService.getFallbackLayer(annotationId, oldTracingId) // remote wk does not know the new id yet + tracing <- tracingService.find(annotationId, tracingId) ?~> "tracing.notFound" segmentIndexBuffer = new VolumeSegmentIndexBuffer(tracingId, volumeSegmentIndexClient, tracing.version, tracingService.remoteDatastoreClient, fallbackLayer, dataLayer.additionalAxes, - userToken) + tc) _ <- Fox.serialCombined(updatedBucketsMutable.toList) { bucketPosition: BucketPosition => for { _ <- saveBucket(dataLayer, bucketPosition, bucketDataMapMutable(bucketPosition), tracing.version) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingLayer.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingLayer.scala index f8d620e1405..e22563365ee 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingLayer.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingLayer.scala @@ -1,5 +1,6 @@ package com.scalableminds.webknossos.tracingstore.tracings.volume +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.geometry.{BoundingBox, Vec3Int} import com.scalableminds.util.tools.{Fox, FoxImplicits} @@ -75,7 +76,7 @@ case class VolumeTracingLayer( isTemporaryTracing: Boolean = false, includeFallbackDataIfAvailable: Boolean = false, tracing: VolumeTracing, - userToken: Option[String], + tokenContext: TokenContext, additionalAxes: Option[Seq[AdditionalAxis]] )(implicit val volumeDataStore: FossilDBClient, implicit val volumeDataCache: TemporaryVolumeDataStore, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index b20bf1525ff..6f54ca67529 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -1,6 +1,7 @@ package com.scalableminds.webknossos.tracingstore.tracings.volume import com.google.inject.Inject +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int} import com.scalableminds.util.io.{NamedStream, ZipIO} @@ -122,14 +123,13 @@ class VolumeTracingService @Inject()( def applyBucketMutatingActions(annotationId: String, updateActions: List[BucketMutatingVolumeUpdateAction], - newVersion: Long, - userToken: Option[String]): Fox[Unit] = + newVersion: Long)(implicit tc: TokenContext): Fox[Unit] = for { // warning, may be called multiple times with the same version number (due to transaction management). // frontend ensures that each bucket is only updated once per transaction tracingId <- updateActions.headOption.map(_.actionTracingId).toFox - fallbackLayerOpt <- getFallbackLayer(annotationId, tracingId, userToken) - tracing <- find(annotationId, tracingId, userToken = userToken) ?~> "tracing.notFound" + fallbackLayerOpt <- getFallbackLayer(annotationId, tracingId) + tracing <- find(annotationId, tracingId) ?~> "tracing.notFound" segmentIndexBuffer = new VolumeSegmentIndexBuffer( tracingId, volumeSegmentIndexClient, @@ -137,7 +137,7 @@ class VolumeTracingService @Inject()( remoteDatastoreClient, fallbackLayerOpt, AdditionalAxis.fromProtosAsOpt(tracing.additionalAxes), - userToken + tc ) _ <- Fox.serialCombined(updateActions) { case a: UpdateBucketVolumeAction => @@ -150,13 +150,7 @@ class VolumeTracingService @Inject()( if (!tracing.getHasSegmentIndex) { Fox.failure("Cannot delete segment data for annotations without segment index.") } else - deleteSegmentData(annotationId, - tracingId, - tracing, - a, - segmentIndexBuffer, - newVersion, - userToken = userToken) ?~> "Failed to delete segment data." + deleteSegmentData(annotationId, tracingId, tracing, a, segmentIndexBuffer, newVersion) ?~> "Failed to delete segment data." case _ => Fox.failure("Unknown bucket-mutating action.") } _ <- segmentIndexBuffer.flush() @@ -166,7 +160,7 @@ class VolumeTracingService @Inject()( volumeTracing: VolumeTracing, action: UpdateBucketVolumeAction, segmentIndexBuffer: VolumeSegmentIndexBuffer, - updateGroupVersion: Long): Fox[VolumeTracing] = + updateGroupVersion: Long)(implicit tc: TokenContext): Fox[VolumeTracing] = for { _ <- assertMagIsValid(volumeTracing, action.mag) ?~> s"Received a mag-${action.mag.toMagLiteral(allowScalar = true)} bucket, which is invalid for this annotation." bucketPosition = BucketPosition(action.position.x, @@ -199,13 +193,12 @@ class VolumeTracingService @Inject()( tracingId: String, version: Option[Long] = None, useCache: Boolean = true, - applyUpdates: Boolean = false, - userToken: Option[String]): Fox[VolumeTracing] = + applyUpdates: Boolean = false)(implicit tc: TokenContext): Fox[VolumeTracing] = if (tracingId == TracingIds.dummyTracingId) Fox.successful(dummyTracing) else { for { - annotation <- annotationService.getWithTracings(annotationId, version, List.empty, List(tracingId), userToken) // TODO is applyUpdates still needed? + annotation <- annotationService.getWithTracings(annotationId, version, List.empty, List(tracingId)) // TODO is applyUpdates still needed? tracing <- annotation.getVolume(tracingId) } yield tracing } @@ -223,8 +216,7 @@ class VolumeTracingService @Inject()( volumeTracing: VolumeTracing, a: DeleteSegmentDataVolumeAction, segmentIndexBuffer: VolumeSegmentIndexBuffer, - version: Long, - userToken: Option[String]): Fox[VolumeTracing] = + version: Long)(implicit tc: TokenContext): Fox[VolumeTracing] = for { _ <- Fox.successful(()) dataLayer = volumeTracingLayer(tracingId, volumeTracing) @@ -239,7 +231,7 @@ class VolumeTracingService @Inject()( Fox.serialCombined(additionalCoordinateList)(additionalCoordinates => { val mag = vec3IntFromProto(resolution) for { - fallbackLayer <- getFallbackLayer(annotationId, tracingId, userToken) + fallbackLayer <- getFallbackLayer(annotationId, tracingId) bucketPositionsRaw <- volumeSegmentIndexService.getSegmentToBucketIndexWithEmptyFallbackWithoutBuffer( fallbackLayer, tracingId, @@ -249,8 +241,7 @@ class VolumeTracingService @Inject()( mappingName, editableMappingTracingId(volumeTracing, tracingId), additionalCoordinates, - dataLayer.additionalAxes, - userToken + dataLayer.additionalAxes ) bucketPositions = bucketPositionsRaw.values .map(vec3IntFromProto) @@ -294,22 +285,21 @@ class VolumeTracingService @Inject()( tracingId: String, sourceVersion: Long, newVersion: Long, - tracing: VolumeTracing, - userToken: Option[String]): Fox[VolumeTracing] = { + tracing: VolumeTracing)(implicit tc: TokenContext): Fox[VolumeTracing] = { val dataLayer = volumeTracingLayer(tracingId, tracing) val bucketStream = dataLayer.volumeBucketProvider.bucketStreamWithVersion() for { - fallbackLayer <- getFallbackLayer(annotationId, tracingId, userToken) + fallbackLayer <- getFallbackLayer(annotationId, tracingId) segmentIndexBuffer = new VolumeSegmentIndexBuffer(tracingId, volumeSegmentIndexClient, newVersion, remoteDatastoreClient, fallbackLayer, dataLayer.additionalAxes, - userToken) - sourceTracing <- find(annotationId, tracingId, Some(sourceVersion), userToken = userToken) + tc) + sourceTracing <- find(annotationId, tracingId, Some(sourceVersion)) mappingName <- baseMappingName(sourceTracing) _ <- Fox.serialCombined(bucketStream) { case (bucketPosition, dataBeforeRevert, version) => @@ -352,11 +342,9 @@ class VolumeTracingService @Inject()( } yield sourceTracing } - def initializeWithDataMultiple(annotationId: String, - tracingId: String, - tracing: VolumeTracing, - initialData: File, - userToken: Option[String])(implicit mp: MessagesProvider): Fox[Set[Vec3Int]] = + def initializeWithDataMultiple(annotationId: String, tracingId: String, tracing: VolumeTracing, initialData: File)( + implicit mp: MessagesProvider, + tc: TokenContext): Fox[Set[Vec3Int]] = if (tracing.version != 0L) Failure("Tracing has already been edited.") else { @@ -393,7 +381,7 @@ class VolumeTracingService @Inject()( mergedVolume.largestSegmentId.toLong, tracing.elementClass) destinationDataLayer = volumeTracingLayer(tracingId, tracing) - fallbackLayer <- getFallbackLayer(annotationId, tracingId, userToken) + fallbackLayer <- getFallbackLayer(annotationId, tracingId) segmentIndexBuffer = new VolumeSegmentIndexBuffer( tracingId, volumeSegmentIndexClient, @@ -401,7 +389,7 @@ class VolumeTracingService @Inject()( remoteDatastoreClient, fallbackLayer, AdditionalAxis.fromProtosAsOpt(tracing.additionalAxes), - userToken + tc ) _ <- mergedVolume.withMergedBuckets { (bucketPosition, bytes) => for { @@ -427,15 +415,14 @@ class VolumeTracingService @Inject()( tracingId: String, tracing: VolumeTracing, initialData: File, - resolutionRestrictions: ResolutionRestrictions, - userToken: Option[String]): Fox[Set[Vec3Int]] = + resolutionRestrictions: ResolutionRestrictions)(implicit tc: TokenContext): Fox[Set[Vec3Int]] = if (tracing.version != 0L) { Failure("Tracing has already been edited.") } else { val dataLayer = volumeTracingLayer(tracingId, tracing) val savedResolutions = new mutable.HashSet[Vec3Int]() for { - fallbackLayer <- getFallbackLayer(annotationId, tracingId, userToken) + fallbackLayer <- getFallbackLayer(annotationId, tracingId) mappingName <- baseMappingName(tracing) segmentIndexBuffer = new VolumeSegmentIndexBuffer( tracingId, @@ -444,7 +431,7 @@ class VolumeTracingService @Inject()( remoteDatastoreClient, fallbackLayer, AdditionalAxis.fromProtosAsOpt(tracing.additionalAxes), - userToken + tc ) _ <- withBucketsFromZip(initialData) { (bucketPosition, bytes) => if (resolutionRestrictions.isForbidden(bucketPosition.mag)) { @@ -474,10 +461,11 @@ class VolumeTracingService @Inject()( } } - def allDataZip(tracingId: String, - tracing: VolumeTracing, - volumeDataZipFormat: VolumeDataZipFormat, - voxelSize: Option[VoxelSize])(implicit ec: ExecutionContext): Fox[Files.TemporaryFile] = { + def allDataZip( + tracingId: String, + tracing: VolumeTracing, + volumeDataZipFormat: VolumeDataZipFormat, + voxelSize: Option[VoxelSize])(implicit ec: ExecutionContext, tc: TokenContext): Fox[Files.TemporaryFile] = { val zipped = temporaryFileCreator.create(tracingId, ".zip") val os = new BufferedOutputStream(new FileOutputStream(new File(zipped.path.toString))) allDataToOutputStream(tracingId, tracing, volumeDataZipFormat, voxelSize, os).map(_ => zipped) @@ -487,7 +475,7 @@ class VolumeTracingService @Inject()( tracing: VolumeTracing, volumeDataZipFormmat: VolumeDataZipFormat, voxelSize: Option[VoxelSize], - os: OutputStream)(implicit ec: ExecutionContext): Fox[Unit] = { + os: OutputStream)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Unit] = { val dataLayer = volumeTracingLayer(tracingId, tracing) val buckets: Iterator[NamedStream] = volumeDataZipFormmat match { case VolumeDataZipFormat.wkw => @@ -518,11 +506,10 @@ class VolumeTracingService @Inject()( def data(tracingId: String, tracing: VolumeTracing, dataRequests: DataRequestCollection, - includeFallbackDataIfAvailable: Boolean = false, - userToken: Option[String] = None): Fox[(Array[Byte], List[Int])] = + includeFallbackDataIfAvailable: Boolean = false)(implicit tc: TokenContext): Fox[(Array[Byte], List[Int])] = for { isTemporaryTracing <- isTemporaryTracing(tracingId) - dataLayer = volumeTracingLayer(tracingId, tracing, isTemporaryTracing, includeFallbackDataIfAvailable, userToken) + dataLayer = volumeTracingLayer(tracingId, tracing, isTemporaryTracing, includeFallbackDataIfAvailable) requests = dataRequests.map(r => DataServiceDataRequest(null, dataLayer, r.cuboid(dataLayer), r.settings.copy(appliedAgglomerate = None))) data <- binaryDataService.handleDataRequests(requests) @@ -537,13 +524,12 @@ class VolumeTracingService @Inject()( editPosition: Option[Vec3Int], editRotation: Option[Vec3Double], boundingBox: Option[BoundingBox], - mappingName: Option[String], - userToken: Option[String]): Fox[(String, VolumeTracing)] = { + mappingName: Option[String])(implicit tc: TokenContext): Fox[(String, VolumeTracing)] = { val tracingWithBB = addBoundingBoxFromTaskIfRequired(sourceTracing, fromTask, datasetBoundingBox) val tracingWithResolutionRestrictions = restrictMagList(tracingWithBB, resolutionRestrictions) for { - fallbackLayer <- getFallbackLayer(annotationId, tracingId, userToken) - hasSegmentIndex <- VolumeSegmentIndexService.canHaveSegmentIndex(remoteDatastoreClient, fallbackLayer, userToken) + fallbackLayer <- getFallbackLayer(annotationId, tracingId) + hasSegmentIndex <- VolumeSegmentIndexService.canHaveSegmentIndex(remoteDatastoreClient, fallbackLayer) newTracing = tracingWithResolutionRestrictions.copy( createdTimestamp = System.currentTimeMillis(), editPosition = editPosition.map(vec3IntToProto).getOrElse(tracingWithResolutionRestrictions.editPosition), @@ -556,7 +542,7 @@ class VolumeTracingService @Inject()( ) _ <- bool2Fox(newTracing.resolutions.nonEmpty) ?~> "resolutionRestrictions.tooTight" newId <- save(newTracing, None, newTracing.version) - _ <- duplicateData(annotationId, tracingId, sourceTracing, newId, newTracing, userToken) + _ <- duplicateData(annotationId, tracingId, sourceTracing, newId, newTracing) } yield (newId, newTracing) } @@ -580,14 +566,13 @@ class VolumeTracingService @Inject()( sourceId: String, sourceTracing: VolumeTracing, destinationId: String, - destinationTracing: VolumeTracing, - userToken: Option[String]): Fox[Unit] = + destinationTracing: VolumeTracing)(implicit tc: TokenContext): Fox[Unit] = for { isTemporaryTracing <- isTemporaryTracing(sourceId) sourceDataLayer = volumeTracingLayer(sourceId, sourceTracing, isTemporaryTracing) buckets: Iterator[(BucketPosition, Array[Byte])] = sourceDataLayer.bucketProvider.bucketStream() destinationDataLayer = volumeTracingLayer(destinationId, destinationTracing) - fallbackLayer <- getFallbackLayer(annotationId, sourceId, userToken) + fallbackLayer <- getFallbackLayer(annotationId, sourceId) segmentIndexBuffer = new VolumeSegmentIndexBuffer( destinationId, volumeSegmentIndexClient, @@ -595,7 +580,7 @@ class VolumeTracingService @Inject()( remoteDatastoreClient, fallbackLayer, AdditionalAxis.fromProtosAsOpt(sourceTracing.additionalAxes), - userToken + tc ) mappingName <- baseMappingName(sourceTracing) _ <- Fox.serialCombined(buckets) { @@ -619,18 +604,18 @@ class VolumeTracingService @Inject()( _ <- segmentIndexBuffer.flush() } yield () - private def volumeTracingLayer(tracingId: String, - tracing: VolumeTracing, - isTemporaryTracing: Boolean = false, - includeFallbackDataIfAvailable: Boolean = false, - userToken: Option[String] = None): VolumeTracingLayer = + private def volumeTracingLayer( + tracingId: String, + tracing: VolumeTracing, + isTemporaryTracing: Boolean = false, + includeFallbackDataIfAvailable: Boolean = false)(implicit tc: TokenContext): VolumeTracingLayer = VolumeTracingLayer( name = tracingId, isTemporaryTracing = isTemporaryTracing, volumeTracingService = this, includeFallbackDataIfAvailable = includeFallbackDataIfAvailable, tracing = tracing, - userToken = userToken, + tokenContext = tc, additionalAxes = AdditionalAxis.fromProtosAsOpt(tracing.additionalAxes) ) @@ -665,35 +650,26 @@ class VolumeTracingService @Inject()( toCache) } yield id - def downsample(annotationId: String, - tracingId: String, - oldTracingId: String, - tracing: VolumeTracing, - userToken: Option[String]): Fox[Unit] = + def downsample(annotationId: String, tracingId: String, oldTracingId: String, tracing: VolumeTracing)( + implicit tc: TokenContext): Fox[Unit] = for { resultingResolutions <- downsampleWithLayer(annotationId, tracingId, oldTracingId, tracing, volumeTracingLayer(tracingId, tracing), - this, - userToken) + this) _ <- updateResolutionList(tracingId, tracing, resultingResolutions.toSet) } yield () def volumeBucketsAreEmpty(tracingId: String): Boolean = volumeDataStore.getMultipleKeys(None, Some(tracingId), limit = Some(1))(toBox).isEmpty - def createAdHocMesh(annotationId: String, - tracingId: String, - request: WebknossosAdHocMeshRequest, - userToken: Option[String]): Fox[(Array[Float], List[Int])] = + def createAdHocMesh(annotationId: String, tracingId: String, request: WebknossosAdHocMeshRequest)( + implicit tc: TokenContext): Fox[(Array[Float], List[Int])] = for { - tracing <- find(annotationId: String, tracingId, userToken = userToken) ?~> "tracing.notFound" - segmentationLayer = volumeTracingLayer(tracingId, - tracing, - includeFallbackDataIfAvailable = true, - userToken = userToken) + tracing <- find(annotationId: String, tracingId) ?~> "tracing.notFound" + segmentationLayer = volumeTracingLayer(tracingId, tracing, includeFallbackDataIfAvailable = true) adHocMeshRequest = AdHocMeshRequest( None, segmentationLayer, @@ -708,9 +684,9 @@ class VolumeTracingService @Inject()( result <- adHocMeshService.requestAdHocMeshViaActor(adHocMeshRequest) } yield result - def findData(annotationId: String, tracingId: String, userToken: Option[String]): Fox[Option[Vec3Int]] = + def findData(annotationId: String, tracingId: String)(implicit tc: TokenContext): Fox[Option[Vec3Int]] = for { - tracing <- find(annotationId: String, tracingId, userToken = userToken) ?~> "tracing.notFound" + tracing <- find(annotationId: String, tracingId) ?~> "tracing.notFound" volumeLayer = volumeTracingLayer(tracingId, tracing) bucketStream = volumeLayer.bucketProvider.bucketStream(Some(tracing.version)) bucketPosOpt = if (bucketStream.hasNext) { @@ -794,8 +770,8 @@ class VolumeTracingService @Inject()( case (None, None) => None } - private def bucketStreamFromSelector(selector: TracingSelector, - tracing: VolumeTracing): Iterator[(BucketPosition, Array[Byte])] = { + private def bucketStreamFromSelector(selector: TracingSelector, tracing: VolumeTracing)( + implicit tc: TokenContext): Iterator[(BucketPosition, Array[Byte])] = { val dataLayer = volumeTracingLayer(selector.tracingId, tracing) dataLayer.bucketProvider.bucketStream(Some(tracing.version)) } @@ -804,8 +780,7 @@ class VolumeTracingService @Inject()( tracings: Seq[VolumeTracing], newId: String, newVersion: Long, - toCache: Boolean, - userToken: Option[String])(implicit mp: MessagesProvider): Fox[MergedVolumeStats] = { + toCache: Boolean)(implicit mp: MessagesProvider, tc: TokenContext): Fox[MergedVolumeStats] = { val elementClass = tracings.headOption.map(_.elementClass).getOrElse(elementClassToProto(ElementClass.uint8)) val resolutionSets = new mutable.HashSet[Set[Vec3Int]]() @@ -856,14 +831,14 @@ class VolumeTracingService @Inject()( elementClass) mergedAdditionalAxes <- Fox.box2Fox(AdditionalAxis.mergeAndAssertSameAdditionalAxes(tracings.map(t => AdditionalAxis.fromProtosAsOpt(t.additionalAxes)))) - fallbackLayer <- getFallbackLayer("dummyAnnotationId", tracingSelectors.head.tracingId, userToken) // TODO annotation id from selectors + fallbackLayer <- getFallbackLayer("dummyAnnotationId", tracingSelectors.head.tracingId) // TODO annotation id from selectors segmentIndexBuffer = new VolumeSegmentIndexBuffer(newId, volumeSegmentIndexClient, newVersion, remoteDatastoreClient, fallbackLayer, mergedAdditionalAxes, - userToken) + tc) _ <- mergedVolume.withMergedBuckets { (bucketPosition, bucketBytes) => for { _ <- saveBucket(newId, elementClass, bucketPosition, bucketBytes, newVersion, toCache, mergedAdditionalAxes) @@ -886,14 +861,13 @@ class VolumeTracingService @Inject()( tracingId: String, tracing: VolumeTracing, currentVersion: Long, - userToken: Option[String], - dryRun: Boolean): Fox[Option[Int]] = { + dryRun: Boolean)(implicit tc: TokenContext): Fox[Option[Int]] = { var processedBucketCount = 0 for { isTemporaryTracing <- isTemporaryTracing(tracingId) sourceDataLayer = volumeTracingLayer(tracingId, tracing, isTemporaryTracing) buckets: Iterator[(BucketPosition, Array[Byte])] = sourceDataLayer.bucketProvider.bucketStream() - fallbackLayer <- getFallbackLayer(annotationId, tracingId, userToken) + fallbackLayer <- getFallbackLayer(annotationId, tracingId) mappingName <- baseMappingName(tracing) segmentIndexBuffer = new VolumeSegmentIndexBuffer(tracingId, volumeSegmentIndexClient, @@ -901,7 +875,7 @@ class VolumeTracingService @Inject()( remoteDatastoreClient, fallbackLayer, sourceDataLayer.additionalAxes, - userToken) + tc) _ <- Fox.serialCombined(buckets) { case (bucketPosition, bucketData) => processedBucketCount += 1 @@ -929,14 +903,12 @@ class VolumeTracingService @Inject()( } yield Some(processedBucketCount) } - def checkIfSegmentIndexMayBeAdded(tracingId: String, tracing: VolumeTracing, userToken: Option[String])( - implicit ec: ExecutionContext): Fox[Boolean] = + def checkIfSegmentIndexMayBeAdded(tracingId: String, tracing: VolumeTracing)(implicit ec: ExecutionContext, + tc: TokenContext): Fox[Boolean] = for { fallbackLayerOpt <- Fox.runIf(tracing.fallbackLayer.isDefined)( remoteFallbackLayerFromVolumeTracing(tracing, tracingId)) - canHaveSegmentIndex <- VolumeSegmentIndexService.canHaveSegmentIndex(remoteDatastoreClient, - fallbackLayerOpt, - userToken) + canHaveSegmentIndex <- VolumeSegmentIndexService.canHaveSegmentIndex(remoteDatastoreClient, fallbackLayerOpt) alreadyHasSegmentIndex = tracing.hasSegmentIndex.getOrElse(false) } yield canHaveSegmentIndex && !alreadyHasSegmentIndex @@ -944,8 +916,7 @@ class VolumeTracingService @Inject()( tracingId: String, tracing: VolumeTracing, zipFile: File, - currentVersion: Int, - userToken: Option[String])(implicit mp: MessagesProvider): Fox[Long] = + currentVersion: Int)(implicit mp: MessagesProvider, tc: TokenContext): Fox[Long] = if (currentVersion != tracing.version) Fox.failure("version.mismatch") else { @@ -972,7 +943,7 @@ class VolumeTracingService @Inject()( mergedVolume.largestSegmentId.toLong, tracing.elementClass) dataLayer = volumeTracingLayer(tracingId, tracing) - fallbackLayer <- getFallbackLayer(annotationId, tracingId, userToken) + fallbackLayer <- getFallbackLayer(annotationId, tracingId) mappingName <- baseMappingName(tracing) segmentIndexBuffer <- Fox.successful( new VolumeSegmentIndexBuffer(tracingId, @@ -981,7 +952,7 @@ class VolumeTracingService @Inject()( remoteDatastoreClient, fallbackLayer, dataLayer.additionalAxes, - userToken)) + tc)) _ <- mergedVolume.withMergedBuckets { (bucketPosition, bucketBytes) => for { _ <- saveBucket(volumeLayer, bucketPosition, bucketBytes, tracing.version + 1) @@ -1020,7 +991,7 @@ class VolumeTracingService @Inject()( def dummyTracing: VolumeTracing = ??? - def mergeEditableMappings(tracingsWithIds: List[(VolumeTracing, String)], userToken: Option[String]): Fox[String] = + def mergeEditableMappings(tracingsWithIds: List[(VolumeTracing, String)])(implicit tc: TokenContext): Fox[String] = if (tracingsWithIds.forall(tracingWithId => tracingWithId._1.getHasEditableMapping)) { for { remoteFallbackLayers <- Fox.serialCombined(tracingsWithIds)(tracingWithId => @@ -1029,7 +1000,7 @@ class VolumeTracingService @Inject()( _ <- bool2Fox(remoteFallbackLayers.forall(_ == remoteFallbackLayer)) ?~> "Cannot merge editable mappings based on different dataset layers" editableMappingIds <- Fox.serialCombined(tracingsWithIds)(tracingWithId => tracingWithId._1.mappingName) _ <- bool2Fox(editableMappingIds.length == tracingsWithIds.length) ?~> "Not all volume tracings have editable mappings" - newEditableMappingId <- editableMappingService.merge(editableMappingIds, remoteFallbackLayer, userToken) + newEditableMappingId <- editableMappingService.merge(editableMappingIds, remoteFallbackLayer) } yield newEditableMappingId } else if (tracingsWithIds.forall(tracingWithId => !tracingWithId._1.getHasEditableMapping)) { Fox.empty @@ -1037,16 +1008,15 @@ class VolumeTracingService @Inject()( Fox.failure("Cannot merge tracings with and without editable mappings") } - def getFallbackLayer(annotationId: String, - tracingId: String, - userToken: Option[String]): Fox[Option[RemoteFallbackLayer]] = - fallbackLayerCache.getOrLoad((annotationId, tracingId, userToken), - t => getFallbackLayerFromWebknossos(t._1, t._2, t._3)) + def getFallbackLayer(annotationId: String, tracingId: String)( + implicit tc: TokenContext): Fox[Option[RemoteFallbackLayer]] = + fallbackLayerCache.getOrLoad((annotationId, tracingId, tc.userTokenOpt), + t => getFallbackLayerFromWebknossos(t._1, t._2)) - private def getFallbackLayerFromWebknossos(annotationId: String, tracingId: String, userToken: Option[String]) = + private def getFallbackLayerFromWebknossos(annotationId: String, tracingId: String)(implicit tc: TokenContext) = Fox[Option[RemoteFallbackLayer]] { for { - tracing <- find(annotationId, tracingId, userToken = userToken) + tracing <- find(annotationId, tracingId) dataSource <- remoteWebknossosClient.getDataSourceForTracing(tracingId) dataSourceId = dataSource.id fallbackLayerName = tracing.fallbackLayer diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 1c7a3c5ab00..9e9a44a8253 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -5,6 +5,7 @@ # Health endpoint GET /health @com.scalableminds.webknossos.tracingstore.controllers.Application.health +# Annotations (concerns AnnotationProto, not annotation info as stored in postgres) POST /annotation/save @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.save(token: Option[String], annotationId: String) GET /annotation/:annotationId @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.get(token: Option[String], annotationId: String, version: Option[Long]) POST /annotation/:annotationId/update @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.update(token: Option[String], annotationId: String) From de717c70d2d1902f457fa9aed0eee58d97da7bea Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 17 Sep 2024 14:30:43 +0200 Subject: [PATCH 061/150] use tracingId directly for editableMappings --- .../annotation/AnnotationWithTracings.scala | 6 ++ .../annotation/TSAnnotationService.scala | 23 ++++++- .../controllers/TracingController.scala | 10 +-- .../controllers/VolumeTracingController.scala | 15 ++--- .../tracings/TracingService.scala | 3 +- .../EditableMappingService.scala | 65 +++++++++---------- .../skeleton/SkeletonTracingService.scala | 3 +- .../volume/VolumeTracingService.scala | 12 ++-- 8 files changed, 84 insertions(+), 53 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index 2a1eda1d2dc..aad111d9e9b 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -39,6 +39,12 @@ case class AnnotationWithTracings( } } yield volumeTracing + def volumesIdsThatHaveEditableMapping: List[String] = + tracingsById.view.flatMap { + case (id, Right(vt: VolumeTracing)) if vt.getHasEditableMapping => Some(id) + case _ => None + }.toList + def getEditableMappingInfo(tracingId: String): Box[EditableMappingInfo] = for { (info, _) <- editableMappingsByTracingId.get(tracingId) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index c0b5b191620..1d472a89cf5 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -9,6 +9,7 @@ import com.scalableminds.webknossos.datastore.EditableMappingInfo.EditableMappin import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ + EditableMappingService, EditableMappingUpdateAction, EditableMappingUpdater } @@ -150,9 +151,29 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl updates, requestedSkeletonTracingIds, requestedVolumeTracingIds) ?~> "findTracingsForUpdates.failed" - updated <- applyUpdates(annotationWithTracings, annotationId, updates, targetVersion) ?~> "applyUpdates.inner.failed" + annotationWithTracingsAndMappings <- findEditableMappingsForUpdates(annotationWithTracings, updates) + updated <- applyUpdates(annotationWithTracingsAndMappings, annotationId, updates, targetVersion) ?~> "applyUpdates.inner.failed" } yield updated + private def findEditableMappingsForUpdates( // TODO integrate with findTracings? + annotationWithTracings: AnnotationWithTracings, + updates: List[UpdateAction])(implicit ec: ExecutionContext) = { + val editableMappingIds = annotationWithTracings.volumesIdsThatHaveEditableMapping + // TODO intersect with editable mapping updates? + for { + editableMappingInfos <- Fox.serialCombined(editableMappingIds) { editableMappingId => + tracingDataStore.editableMappingsInfo.get(editableMappingId, version = Some(annotationWithTracings.version))( + fromProtoBytes[EditableMappingInfo]) + } + } yield + annotationWithTracings.copy( + editableMappingsByTracingId = editableMappingInfos + .map(keyValuePair => (keyValuePair.key, (keyValuePair.value, editableMappingUpdaterFor(keyValuePair.value)))) + .toMap) + } + + def editableMappingUpdaterFor(editableMappingInfo: EditableMappingInfo): EditableMappingUpdater = ??? // TODO + private def findTracingsForUpdates( annotation: AnnotationProto, updates: List[UpdateAction], diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala index 133453f00b8..f157621f7ad 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala @@ -106,13 +106,13 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C case (Some(tracing), Some(selector)) => Some((tracing, selector.tracingId)) case _ => None } - newId = tracingService.generateTracingId + newTracingId = tracingService.generateTracingId mergedVolumeStats <- tracingService.mergeVolumeData(request.body.flatten, tracingsWithIds.map(_._1), - newId, + newTracingId, newVersion = 0L, toCache = !persist) - newEditableMappingIdBox <- tracingService.mergeEditableMappings(tracingsWithIds).futureBox + newEditableMappingIdBox <- tracingService.mergeEditableMappings(newTracingId, tracingsWithIds).futureBox newEditableMappingIdOpt <- newEditableMappingIdBox match { case Full(newEditableMappingId) => Fox.successful(Some(newEditableMappingId)) case Empty => Fox.successful(None) @@ -120,8 +120,8 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C } mergedTracing <- Fox.box2Fox( tracingService.merge(tracingsWithIds.map(_._1), mergedVolumeStats, newEditableMappingIdOpt)) - _ <- tracingService.save(mergedTracing, Some(newId), version = 0, toCache = !persist) - } yield Ok(Json.toJson(newId)) + _ <- tracingService.save(mergedTracing, Some(newTracingId), version = 0, toCache = !persist) + } yield Ok(Json.toJson(newTracingId)) } } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 4eb936fc829..ca53733ba43 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -221,11 +221,13 @@ class VolumeTracingController @Inject()( boundingBoxParsed <- Fox.runOptional(boundingBox)(BoundingBox.fromLiteral) remoteFallbackLayerOpt <- Fox.runIf(tracing.getHasEditableMapping)( tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId)) - newEditableMappingId <- Fox.runIf(tracing.getHasEditableMapping)( - editableMappingService.duplicate(tracing.mappingName, version = None, remoteFallbackLayerOpt)) + newTracingId = tracingService.generateTracingId + _ <- Fox.runIf(tracing.getHasEditableMapping)(editableMappingService + .duplicate(tracing.mappingName, newTracingId, version = None, remoteFallbackLayerOpt)) (newId, newTracing) <- tracingService.duplicate( annotationId, tracingId, + newTracingId, tracing, fromTask.getOrElse(false), datasetBoundingBox, @@ -233,7 +235,7 @@ class VolumeTracingController @Inject()( editPositionParsed, editRotationParsed, boundingBoxParsed, - newEditableMappingId + mappingName = None ) _ <- Fox.runIfOptionTrue(downsample)(tracingService.downsample(annotationId, newId, tracingId, newTracing)) } yield Ok(Json.toJson(newId)) @@ -377,9 +379,8 @@ class VolumeTracingController @Inject()( tracingMappingName <- tracing.mappingName ?~> "annotation.noMappingSet" _ <- assertMappingIsNotLocked(tracing) _ <- bool2Fox(tracingService.volumeBucketsAreEmpty(tracingId)) ?~> "annotation.volumeBucketsNotEmpty" - (editableMappingId, editableMappingInfo) <- editableMappingService.create( - baseMappingName = tracingMappingName) - volumeUpdate = UpdateMappingNameVolumeAction(Some(editableMappingId), + editableMappingInfo <- editableMappingService.create(tracingId, baseMappingName = tracingMappingName) + volumeUpdate = UpdateMappingNameVolumeAction(Some(tracingId), isEditable = Some(true), isLocked = Some(true), actionTracingId = tracingId, @@ -399,7 +400,6 @@ class VolumeTracingController @Inject()( 0)) ) infoJson <- editableMappingService.infoJson(tracingId = tracingId, - editableMappingId = editableMappingId, editableMappingInfo = editableMappingInfo, version = Some(0L)) } yield Ok(infoJson) @@ -480,7 +480,6 @@ class VolumeTracingController @Inject()( mappingName <- tracing.mappingName.toFox editableMappingInfo <- editableMappingService.getInfoNEW(annotationId, tracingId, version) infoJson <- editableMappingService.infoJson(tracingId = tracingId, - editableMappingId = mappingName, editableMappingInfo = editableMappingInfo, version = version) } yield Ok(infoJson) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala index 9d0291c6c51..ed8428eb7aa 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala @@ -171,5 +171,6 @@ trait TracingService[T <: GeneratedMessage] newVersion: Long, toCache: Boolean)(implicit mp: MessagesProvider, tc: TokenContext): Fox[MergedVolumeStats] - def mergeEditableMappings(tracingsWithIds: List[(T, String)])(implicit tc: TokenContext): Fox[String] + def mergeEditableMappings(newTracingId: String, tracingsWithIds: List[(T, String)])( + implicit tc: TokenContext): Fox[String] } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index e82f75cb1e3..ef8671a611f 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -121,15 +121,12 @@ class EditableMappingService @Inject()( private lazy val agglomerateToGraphCache: AlfuCache[(String, Long, Long), AgglomerateGraph] = AlfuCache(maxCapacity = 50) - def infoJson(tracingId: String, - editableMappingInfo: EditableMappingInfo, - editableMappingId: String, - version: Option[Long]): Fox[JsObject] = + def infoJson(tracingId: String, editableMappingInfo: EditableMappingInfo, version: Option[Long]): Fox[JsObject] = for { - version <- getClosestMaterializableVersionOrZero(editableMappingId, version) + version <- getClosestMaterializableVersionOrZero(tracingId, version) } yield Json.obj( - "mappingName" -> editableMappingId, + "mappingName" -> tracingId, // TODO remove? "version" -> version, "tracingId" -> tracingId, "baseMappingName" -> editableMappingInfo.baseMappingName, @@ -137,36 +134,39 @@ class EditableMappingService @Inject()( "createdTimestamp" -> editableMappingInfo.createdTimestamp ) - def create(baseMappingName: String): Fox[(String, EditableMappingInfo)] = { - val newId = generateId + def create(tracingId: String, baseMappingName: String): Fox[EditableMappingInfo] = { val newEditableMappingInfo = EditableMappingInfo( baseMappingName = baseMappingName, createdTimestamp = Instant.now.epochMillis, largestAgglomerateId = 0L ) for { - _ <- tracingDataStore.editableMappingsInfo.put(newId, 0L, toProtoBytes(newEditableMappingInfo)) - } yield (newId, newEditableMappingInfo) + _ <- tracingDataStore.editableMappingsInfo.put(tracingId, 0L, toProtoBytes(newEditableMappingInfo)) + } yield newEditableMappingInfo } def duplicate(editableMappingIdOpt: Option[String], + newTracingId: String, version: Option[Long], - remoteFallbackLayerBox: Box[RemoteFallbackLayer])(implicit tc: TokenContext): Fox[String] = + remoteFallbackLayerBox: Box[RemoteFallbackLayer])(implicit tc: TokenContext): Fox[Unit] = for { editableMappingId <- editableMappingIdOpt ?~> "duplicate on editable mapping without id" remoteFallbackLayer <- remoteFallbackLayerBox ?~> "duplicate on editable mapping without remote fallback layer" editableMappingInfoAndVersion <- getInfoAndActualVersion(editableMappingId, version, remoteFallbackLayer) - newIdAndInfoV0 <- create(editableMappingInfoAndVersion._1.baseMappingName) - newId = newIdAndInfoV0._1 + newIdAndInfoV0 <- create(newTracingId, editableMappingInfoAndVersion._1.baseMappingName) newVersion = editableMappingInfoAndVersion._2 - _ <- tracingDataStore.editableMappingsInfo.put(newId, newVersion, toProtoBytes(editableMappingInfoAndVersion._1)) - _ <- duplicateSegmentToAgglomerate(editableMappingId, newId, newVersion) - _ <- duplicateAgglomerateToGraph(editableMappingId, newId, newVersion) + _ <- tracingDataStore.editableMappingsInfo.put(newTracingId, + newVersion, + toProtoBytes(editableMappingInfoAndVersion._1)) + _ <- duplicateSegmentToAgglomerate(editableMappingId, newTracingId, newVersion) + _ <- duplicateAgglomerateToGraph(editableMappingId, newTracingId, newVersion) updateActionsWithVersions <- getUpdateActionsWithVersions(editableMappingId, editableMappingInfoAndVersion._2, 0L) _ <- Fox.serialCombined(updateActionsWithVersions) { updateActionsWithVersion: (Long, List[UpdateAction]) => - tracingDataStore.editableMappingUpdates.put(newId, updateActionsWithVersion._1, updateActionsWithVersion._2) + tracingDataStore.editableMappingUpdates.put(newTracingId, + updateActionsWithVersion._1, + updateActionsWithVersion._2) } - } yield newId + } yield () private def duplicateSegmentToAgglomerate(editableMappingId: String, newId: String, newVersion: Long): Fox[Unit] = { val iterator = @@ -689,29 +689,28 @@ class EditableMappingService @Inject()( neighborNodes } - def merge(editableMappingIds: List[String], remoteFallbackLayer: RemoteFallbackLayer)( + def merge(newTracingId: String, tracingIds: List[String], remoteFallbackLayer: RemoteFallbackLayer)( implicit tc: TokenContext): Fox[String] = for { - firstMappingId <- editableMappingIds.headOption.toFox + firstMappingId <- tracingIds.headOption.toFox before = Instant.now - newMappingId <- duplicate(Some(firstMappingId), version = None, Some(remoteFallbackLayer)) - _ <- Fox.serialCombined(editableMappingIds.tail)(editableMappingId => - mergeInto(newMappingId, editableMappingId, remoteFallbackLayer)) - _ = logger.info(s"Merging ${editableMappingIds.length} editable mappings took ${Instant.since(before)}") - } yield newMappingId + _ <- duplicate(Some(firstMappingId), newTracingId, version = None, Some(remoteFallbackLayer)) + _ <- Fox.serialCombined(tracingIds.tail)(editableMappingId => + mergeInto(newTracingId, editableMappingId, remoteFallbackLayer)) + _ = logger.info(s"Merging ${tracingIds.length} editable mappings took ${Instant.since(before)}") + } yield newTracingId // read as: merge source into target (mutate target) - private def mergeInto(targetEditableMappingId: String, - sourceEditableMappingId: String, - remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[Unit] = + private def mergeInto(targetTracingId: String, sourceTracingId: String, remoteFallbackLayer: RemoteFallbackLayer)( + implicit tc: TokenContext): Fox[Unit] = for { - targetNewestVersion <- getClosestMaterializableVersionOrZero(targetEditableMappingId, None) - sourceNewestMaterializedWithVersion <- getInfoAndActualVersion(sourceEditableMappingId, None, remoteFallbackLayer) + targetNewestVersion <- getClosestMaterializableVersionOrZero(targetTracingId, None) + sourceNewestMaterializedWithVersion <- getInfoAndActualVersion(sourceTracingId, None, remoteFallbackLayer) sourceNewestVersion = sourceNewestMaterializedWithVersion._2 - updateActionsWithVersions <- getUpdateActionsWithVersions(sourceEditableMappingId, sourceNewestVersion, 0L) + updateActionsWithVersions <- getUpdateActionsWithVersions(sourceTracingId, sourceNewestVersion, 0L) updateActionsToApply = updateActionsWithVersions.map(_._2).reverse.flatten updater = new EditableMappingUpdater( - targetEditableMappingId, + targetTracingId, sourceNewestMaterializedWithVersion._1.baseMappingName, targetNewestVersion, targetNewestVersion + sourceNewestVersion, @@ -724,7 +723,7 @@ class EditableMappingService @Inject()( ) _ <- updater.applyUpdatesAndSave(sourceNewestMaterializedWithVersion._1, updateActionsToApply) _ <- Fox.serialCombined(updateActionsWithVersions) { updateActionsWithVersion => - tracingDataStore.editableMappingUpdates.put(targetEditableMappingId, + tracingDataStore.editableMappingUpdates.put(targetTracingId, updateActionsWithVersion._1 + targetNewestVersion, updateActionsWithVersion._2) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala index aaff678f9c3..ad512b77dd7 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala @@ -133,6 +133,7 @@ class SkeletonTracingService @Inject()( def dummyTracing: SkeletonTracing = SkeletonTracingDefaults.createInstance - def mergeEditableMappings(tracingsWithIds: List[(SkeletonTracing, String)])(implicit tc: TokenContext): Fox[String] = + def mergeEditableMappings(newTracingId: String, tracingsWithIds: List[(SkeletonTracing, String)])( + implicit tc: TokenContext): Fox[String] = Fox.empty } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 6f54ca67529..4882e4fc21e 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -517,6 +517,7 @@ class VolumeTracingService @Inject()( def duplicate(annotationId: String, tracingId: String, + newTracingId: String, sourceTracing: VolumeTracing, fromTask: Boolean, datasetBoundingBox: Option[BoundingBox], @@ -535,13 +536,15 @@ class VolumeTracingService @Inject()( editPosition = editPosition.map(vec3IntToProto).getOrElse(tracingWithResolutionRestrictions.editPosition), editRotation = editRotation.map(vec3DoubleToProto).getOrElse(tracingWithResolutionRestrictions.editRotation), boundingBox = boundingBoxOptToProto(boundingBox).getOrElse(tracingWithResolutionRestrictions.boundingBox), - mappingName = mappingName.orElse(tracingWithResolutionRestrictions.mappingName), + mappingName = mappingName.orElse( + if (sourceTracing.getHasEditableMapping) Some(newTracingId) + else tracingWithResolutionRestrictions.mappingName), version = 0, // Adding segment index on duplication if the volume tracing allows it. This will be used in duplicateData hasSegmentIndex = Some(hasSegmentIndex) ) _ <- bool2Fox(newTracing.resolutions.nonEmpty) ?~> "resolutionRestrictions.tooTight" - newId <- save(newTracing, None, newTracing.version) + newId <- save(newTracing, Some(newTracingId), newTracing.version) _ <- duplicateData(annotationId, tracingId, sourceTracing, newId, newTracing) } yield (newId, newTracing) } @@ -991,7 +994,8 @@ class VolumeTracingService @Inject()( def dummyTracing: VolumeTracing = ??? - def mergeEditableMappings(tracingsWithIds: List[(VolumeTracing, String)])(implicit tc: TokenContext): Fox[String] = + def mergeEditableMappings(newTracingId: String, tracingsWithIds: List[(VolumeTracing, String)])( + implicit tc: TokenContext): Fox[String] = if (tracingsWithIds.forall(tracingWithId => tracingWithId._1.getHasEditableMapping)) { for { remoteFallbackLayers <- Fox.serialCombined(tracingsWithIds)(tracingWithId => @@ -1000,7 +1004,7 @@ class VolumeTracingService @Inject()( _ <- bool2Fox(remoteFallbackLayers.forall(_ == remoteFallbackLayer)) ?~> "Cannot merge editable mappings based on different dataset layers" editableMappingIds <- Fox.serialCombined(tracingsWithIds)(tracingWithId => tracingWithId._1.mappingName) _ <- bool2Fox(editableMappingIds.length == tracingsWithIds.length) ?~> "Not all volume tracings have editable mappings" - newEditableMappingId <- editableMappingService.merge(editableMappingIds, remoteFallbackLayer) + newEditableMappingId <- editableMappingService.merge(newTracingId, editableMappingIds, remoteFallbackLayer) } yield newEditableMappingId } else if (tracingsWithIds.forall(tracingWithId => !tracingWithId._1.getHasEditableMapping)) { Fox.empty From ba68b72a6eeaef5cef973cb2477e91b9af75f2cc Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 18 Sep 2024 10:13:32 +0200 Subject: [PATCH 062/150] address editable mappings by tracing id --- conf/messages | 1 + .../annotation/TSAnnotationService.scala | 6 +- .../controllers/TracingController.scala | 12 +- .../controllers/VolumeTracingController.scala | 22 ++- .../tracings/TracingService.scala | 2 +- .../EditableMappingLayer.scala | 6 +- .../EditableMappingService.scala | 182 +++++++++--------- .../EditableMappingUpdater.scala | 54 +++--- .../skeleton/SkeletonTracingService.scala | 2 +- .../volume/VolumeTracingService.scala | 14 +- 10 files changed, 146 insertions(+), 155 deletions(-) diff --git a/conf/messages b/conf/messages index ae9f4836e2e..761a81b3f9f 100644 --- a/conf/messages +++ b/conf/messages @@ -187,6 +187,7 @@ annotation.volume.invalidLargestSegmentId=Cannot create tasks with fallback segm annotation.volume.resolutionRestrictionsTooTight=Task type resolution restrictions are too tight, resulting annotation has no resolutions. annotation.volume.resolutionsDoNotMatch=Could not merge volume annotations, as their resolutions differ. Please ensure each annotation has the same set of resolutions. annotation.volume.largestSegmentIdExceedsRange=The largest segment id {0} specified for the annotation layer exceeds the range of its data type {1} +annotation.volume.noEditableMapping=This volume tracing does not have an editable mapping (not a “proofreading” annotation layer) annotation.notFound=Annotation could not be found annotation.notFound.considerLoggingIn=Annotation could not be found. If the annotation is not public, you need to log in to see it. annotation.invalid=Invalid annotation diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 1d472a89cf5..eeb5b8ccc04 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -158,11 +158,11 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl private def findEditableMappingsForUpdates( // TODO integrate with findTracings? annotationWithTracings: AnnotationWithTracings, updates: List[UpdateAction])(implicit ec: ExecutionContext) = { - val editableMappingIds = annotationWithTracings.volumesIdsThatHaveEditableMapping + val volumeIdsWithEditableMapping = annotationWithTracings.volumesIdsThatHaveEditableMapping // TODO intersect with editable mapping updates? for { - editableMappingInfos <- Fox.serialCombined(editableMappingIds) { editableMappingId => - tracingDataStore.editableMappingsInfo.get(editableMappingId, version = Some(annotationWithTracings.version))( + editableMappingInfos <- Fox.serialCombined(volumeIdsWithEditableMapping) { volumeTracingId => + tracingDataStore.editableMappingsInfo.get(volumeTracingId, version = Some(annotationWithTracings.version))( fromProtoBytes[EditableMappingInfo]) } } yield diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala index f157621f7ad..4125161aaf1 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala @@ -112,11 +112,13 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C newTracingId, newVersion = 0L, toCache = !persist) - newEditableMappingIdBox <- tracingService.mergeEditableMappings(newTracingId, tracingsWithIds).futureBox - newEditableMappingIdOpt <- newEditableMappingIdBox match { - case Full(newEditableMappingId) => Fox.successful(Some(newEditableMappingId)) - case Empty => Fox.successful(None) - case f: Failure => f.toFox + mergeEditableMappingsResultBox <- tracingService + .mergeEditableMappings(newTracingId, tracingsWithIds) + .futureBox + newEditableMappingIdOpt <- mergeEditableMappingsResultBox match { + case Full(()) => Fox.successful(Some(newTracingId)) + case Empty => Fox.successful(None) + case f: Failure => f.toFox } mergedTracing <- Fox.box2Fox( tracingService.merge(tracingsWithIds.map(_._1), mergedVolumeStats, newEditableMappingIdOpt)) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index ca53733ba43..3270a11a992 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -222,8 +222,8 @@ class VolumeTracingController @Inject()( remoteFallbackLayerOpt <- Fox.runIf(tracing.getHasEditableMapping)( tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId)) newTracingId = tracingService.generateTracingId - _ <- Fox.runIf(tracing.getHasEditableMapping)(editableMappingService - .duplicate(tracing.mappingName, newTracingId, version = None, remoteFallbackLayerOpt)) + _ <- Fox.runIf(tracing.getHasEditableMapping)( + editableMappingService.duplicate(tracingId, newTracingId, version = None, remoteFallbackLayerOpt)) (newId, newTracing) <- tracingService.duplicate( annotationId, tracingId, @@ -418,7 +418,7 @@ class VolumeTracingController @Inject()( tracing <- tracingService.find(annotationId, tracingId) _ <- bool2Fox(tracing.getHasEditableMapping) ?~> "Mapping is not editable" remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - edges <- editableMappingService.agglomerateGraphMinCut(request.body, remoteFallbackLayer) + edges <- editableMappingService.agglomerateGraphMinCut(tracingId, request.body, remoteFallbackLayer) } yield Ok(Json.toJson(edges)) } } @@ -432,9 +432,11 @@ class VolumeTracingController @Inject()( accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) - _ <- bool2Fox(tracing.getHasEditableMapping) ?~> "Mapping is not editable" + _ <- editableMappingService.assertTracingHasEditableMapping(tracing) remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - (segmentId, edges) <- editableMappingService.agglomerateGraphNeighbors(request.body, remoteFallbackLayer) + (segmentId, edges) <- editableMappingService.agglomerateGraphNeighbors(tracingId, + request.body, + remoteFallbackLayer) } yield Ok(Json.obj("segmentId" -> segmentId, "neighbors" -> Json.toJson(edges))) } } @@ -448,7 +450,7 @@ class VolumeTracingController @Inject()( for { tracing <- tracingService.find(annotationId, tracingId) mappingName <- tracing.mappingName.toFox - _ <- bool2Fox(tracing.getHasEditableMapping) ?~> "Mapping is not editable" + _ <- editableMappingService.assertTracingHasEditableMapping(tracing) currentVersion <- editableMappingService.getClosestMaterializableVersionOrZero(mappingName, None) _ <- bool2Fox(request.body.length == 1) ?~> "Editable mapping update request must contain exactly one update group" updateGroup <- request.body.headOption.toFox @@ -477,7 +479,7 @@ class VolumeTracingController @Inject()( accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) - mappingName <- tracing.mappingName.toFox + _ <- editableMappingService.assertTracingHasEditableMapping(tracing) editableMappingInfo <- editableMappingService.getInfoNEW(annotationId, tracingId, version) infoJson <- editableMappingService.infoJson(tracingId = tracingId, editableMappingInfo = editableMappingInfo, @@ -495,17 +497,17 @@ class VolumeTracingController @Inject()( accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) - editableMappingId <- tracing.mappingName.toFox + _ <- editableMappingService.assertTracingHasEditableMapping(tracing) remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) (editableMappingInfo, editableMappingVersion) <- editableMappingService.getInfoAndActualVersion( - editableMappingId, + tracingId, requestedVersion = None, remoteFallbackLayer = remoteFallbackLayer) relevantMapping: Map[Long, Long] <- editableMappingService.generateCombinedMappingForSegmentIds( request.body.items.toSet, editableMappingInfo, editableMappingVersion, - editableMappingId, + tracingId, remoteFallbackLayer) agglomerateIdsSorted = relevantMapping.toSeq.sortBy(_._1).map(_._2) } yield Ok(ListOfLong(agglomerateIdsSorted).toByteArray) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala index ed8428eb7aa..f6a1d146e0c 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala @@ -172,5 +172,5 @@ trait TracingService[T <: GeneratedMessage] toCache: Boolean)(implicit mp: MessagesProvider, tc: TokenContext): Fox[MergedVolumeStats] def mergeEditableMappings(newTracingId: String, tracingsWithIds: List[(T, String)])( - implicit tc: TokenContext): Fox[String] + implicit tc: TokenContext): Fox[Unit] } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala index f526a4247f2..c46b0ffc44b 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala @@ -30,13 +30,13 @@ class EditableMappingBucketProvider(layer: EditableMappingLayer) extends BucketP override def load(readInstruction: DataReadInstruction)(implicit ec: ExecutionContext): Fox[Array[Byte]] = { val bucket: BucketPosition = readInstruction.bucket for { - editableMappingId <- Fox.successful(layer.name) + tracingId <- Fox.successful(layer.name) _ <- bool2Fox(layer.doesContainBucket(bucket)) remoteFallbackLayer <- layer.editableMappingService .remoteFallbackLayerFromVolumeTracing(layer.tracing, layer.tracingId) // called here to ensure updates are applied (editableMappingInfo, editableMappingVersion) <- layer.editableMappingService.getInfoAndActualVersion( - editableMappingId, + tracingId, requestedVersion = None, remoteFallbackLayer = remoteFallbackLayer)(layer.tokenContext) dataRequest: WebknossosDataRequest = WebknossosDataRequest( @@ -57,7 +57,7 @@ class EditableMappingBucketProvider(layer: EditableMappingLayer) extends BucketP segmentIds, editableMappingInfo, editableMappingVersion, - editableMappingId, + tracingId, remoteFallbackLayer)(layer.tokenContext) mappedData: Array[Byte] <- layer.editableMappingService.mapData(unmappedDataTyped, relevantMapping, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index ef8671a611f..dcf2a0bbfef 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -41,7 +41,6 @@ import play.api.libs.json.{JsObject, Json, OFormat} import java.nio.file.Paths import java.util -import java.util.UUID import scala.concurrent.ExecutionContext import scala.concurrent.duration._ import scala.jdk.CollectionConverters.CollectionHasAsScala @@ -56,15 +55,14 @@ case class MinCutParameters( segmentId1: Long, segmentId2: Long, mag: Vec3Int, - agglomerateId: Long, - editableMappingId: String + agglomerateId: Long ) object MinCutParameters { implicit val jsonFormat: OFormat[MinCutParameters] = Json.format[MinCutParameters] } -case class NeighborsParameters(segmentId: Long, mag: Vec3Int, agglomerateId: Long, editableMappingId: String) +case class NeighborsParameters(segmentId: Long, mag: Vec3Int, agglomerateId: Long) object NeighborsParameters { implicit val jsonFormat: OFormat[NeighborsParameters] = Json.format[NeighborsParameters] @@ -107,8 +105,6 @@ class EditableMappingService @Inject()( val defaultSegmentToAgglomerateChunkSize: Int = 64 * 1024 // max. 1 MiB chunks (two 8-byte numbers per element) - private def generateId: String = UUID.randomUUID.toString - val binaryDataService = new BinaryDataService(Paths.get(""), None, None, None, None, None) adHocMeshServiceHolder.tracingStoreAdHocMeshConfig = (binaryDataService, 30 seconds, 1) private val adHocMeshService: AdHocMeshService = adHocMeshServiceHolder.tracingStoreAdHocMeshService @@ -145,22 +141,17 @@ class EditableMappingService @Inject()( } yield newEditableMappingInfo } - def duplicate(editableMappingIdOpt: Option[String], + def duplicate(sourceTracingId: String, newTracingId: String, version: Option[Long], remoteFallbackLayerBox: Box[RemoteFallbackLayer])(implicit tc: TokenContext): Fox[Unit] = for { - editableMappingId <- editableMappingIdOpt ?~> "duplicate on editable mapping without id" remoteFallbackLayer <- remoteFallbackLayerBox ?~> "duplicate on editable mapping without remote fallback layer" - editableMappingInfoAndVersion <- getInfoAndActualVersion(editableMappingId, version, remoteFallbackLayer) - newIdAndInfoV0 <- create(newTracingId, editableMappingInfoAndVersion._1.baseMappingName) - newVersion = editableMappingInfoAndVersion._2 - _ <- tracingDataStore.editableMappingsInfo.put(newTracingId, - newVersion, - toProtoBytes(editableMappingInfoAndVersion._1)) - _ <- duplicateSegmentToAgglomerate(editableMappingId, newTracingId, newVersion) - _ <- duplicateAgglomerateToGraph(editableMappingId, newTracingId, newVersion) - updateActionsWithVersions <- getUpdateActionsWithVersions(editableMappingId, editableMappingInfoAndVersion._2, 0L) + (duplicatedInfo, newVersion) <- getInfoAndActualVersion(sourceTracingId, version, remoteFallbackLayer) + _ <- tracingDataStore.editableMappingsInfo.put(newTracingId, newVersion, toProtoBytes(duplicatedInfo)) + _ <- duplicateSegmentToAgglomerate(sourceTracingId, newTracingId, newVersion) + _ <- duplicateAgglomerateToGraph(sourceTracingId, newTracingId, newVersion) + updateActionsWithVersions <- getUpdateActionsWithVersions(sourceTracingId, newVersion, 0L) _ <- Fox.serialCombined(updateActionsWithVersions) { updateActionsWithVersion: (Long, List[UpdateAction]) => tracingDataStore.editableMappingUpdates.put(newTracingId, updateActionsWithVersion._1, @@ -168,9 +159,9 @@ class EditableMappingService @Inject()( } } yield () - private def duplicateSegmentToAgglomerate(editableMappingId: String, newId: String, newVersion: Long): Fox[Unit] = { + private def duplicateSegmentToAgglomerate(sourceTracingId: String, newId: String, newVersion: Long): Fox[Unit] = { val iterator = - new VersionedFossilDbIterator(editableMappingId, + new VersionedFossilDbIterator(sourceTracingId, tracingDataStore.editableMappingsSegmentToAgglomerate, Some(newVersion)) for { @@ -186,9 +177,9 @@ class EditableMappingService @Inject()( } yield () } - private def duplicateAgglomerateToGraph(editableMappingId: String, newId: String, newVersion: Long): Fox[Unit] = { + private def duplicateAgglomerateToGraph(sourceTracingId: String, newId: String, newVersion: Long): Fox[Unit] = { val iterator = - new VersionedFossilDbIterator(editableMappingId, + new VersionedFossilDbIterator(sourceTracingId, tracingDataStore.editableMappingsAgglomerateToGraph, Some(newVersion)) for { @@ -209,16 +200,19 @@ class EditableMappingService @Inject()( tracing <- annotation.getEditableMappingInfo(tracingId) } yield tracing - def getInfo(editableMappingId: String, version: Option[Long] = None, remoteFallbackLayer: RemoteFallbackLayer)( + def getInfo(tracingId: String, version: Option[Long] = None, remoteFallbackLayer: RemoteFallbackLayer)( implicit tc: TokenContext): Fox[EditableMappingInfo] = for { - (info, _) <- getInfoAndActualVersion(editableMappingId, version, remoteFallbackLayer) + (info, _) <- getInfoAndActualVersion(tracingId, version, remoteFallbackLayer) } yield info - def getBaseMappingName(editableMappingId: String): Fox[Option[String]] = + def assertTracingHasEditableMapping(tracing: VolumeTracing)(implicit ec: ExecutionContext): Fox[Unit] = + bool2Fox(tracing.getHasEditableMapping) ?~> "annotation.volume.noEditableMapping" + + def getBaseMappingName(tracingId: String): Fox[Option[String]] = for { - desiredVersion <- getClosestMaterializableVersionOrZero(editableMappingId, None) - infoBox <- getClosestMaterialized(editableMappingId, desiredVersion).futureBox + desiredVersion <- getClosestMaterializableVersionOrZero(tracingId, None) + infoBox <- getClosestMaterialized(tracingId, desiredVersion).futureBox } yield infoBox match { case Full(info) => Some(info.value.baseMappingName) @@ -226,34 +220,34 @@ class EditableMappingService @Inject()( } def getInfoAndActualVersion( - editableMappingId: String, + tracingId: String, requestedVersion: Option[Long] = None, remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[(EditableMappingInfo, Long)] = for { - desiredVersion <- getClosestMaterializableVersionOrZero(editableMappingId, requestedVersion) + desiredVersion <- getClosestMaterializableVersionOrZero(tracingId, requestedVersion) materializedInfo <- materializedInfoCache.getOrLoad( - (editableMappingId, desiredVersion), - _ => applyPendingUpdates(editableMappingId, desiredVersion, remoteFallbackLayer)) + (tracingId, desiredVersion), + _ => applyPendingUpdates(tracingId, desiredVersion, remoteFallbackLayer)) } yield (materializedInfo, desiredVersion) - def update(editableMappingId: String, + def update(tracingId: String, updateActionGroup: UpdateActionGroup, newVersion: Long, remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[Unit] = for { actionsWithTimestamp <- Fox.successful(updateActionGroup.actions.map(_.addTimestamp(updateActionGroup.timestamp))) - _ <- dryApplyUpdates(editableMappingId, newVersion, actionsWithTimestamp, remoteFallbackLayer) ?~> "editableMapping.dryUpdate.failed" - _ <- tracingDataStore.editableMappingUpdates.put(editableMappingId, newVersion, actionsWithTimestamp) + _ <- dryApplyUpdates(tracingId, newVersion, actionsWithTimestamp, remoteFallbackLayer) ?~> "editableMapping.dryUpdate.failed" + _ <- tracingDataStore.editableMappingUpdates.put(tracingId, newVersion, actionsWithTimestamp) } yield () - private def dryApplyUpdates(editableMappingId: String, + private def dryApplyUpdates(tracingId: String, newVersion: Long, updates: List[UpdateAction], remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[Unit] = for { - (previousInfo, previousVersion) <- getInfoAndActualVersion(editableMappingId, None, remoteFallbackLayer) + (previousInfo, previousVersion) <- getInfoAndActualVersion(tracingId, None, remoteFallbackLayer) updater = new EditableMappingUpdater( - editableMappingId, + tracingId, previousInfo.baseMappingName, previousVersion, newVersion, @@ -264,20 +258,20 @@ class EditableMappingService @Inject()( tracingDataStore, relyOnAgglomerateIds = updates.length <= 1 ) - updated <- updater.applyUpdatesAndSave(previousInfo, updates, dry = true) ?~> "editableMapping.update.failed" + _ <- updater.applyUpdatesAndSave(previousInfo, updates, dry = true) ?~> "editableMapping.update.failed" } yield () - def applyPendingUpdates(editableMappingId: String, desiredVersion: Long, remoteFallbackLayer: RemoteFallbackLayer)( + def applyPendingUpdates(tracingId: String, desiredVersion: Long, remoteFallbackLayer: RemoteFallbackLayer)( implicit tc: TokenContext): Fox[EditableMappingInfo] = for { - closestMaterializedWithVersion <- getClosestMaterialized(editableMappingId, desiredVersion) + closestMaterializedWithVersion <- getClosestMaterialized(tracingId, desiredVersion) updatedEditableMappingInfo: EditableMappingInfo <- if (desiredVersion == closestMaterializedWithVersion.version) Fox.successful(closestMaterializedWithVersion.value) else for { - pendingUpdates <- getPendingUpdates(editableMappingId, closestMaterializedWithVersion.version, desiredVersion) + pendingUpdates <- getPendingUpdates(tracingId, closestMaterializedWithVersion.version, desiredVersion) updater = new EditableMappingUpdater( - editableMappingId, + tracingId, closestMaterializedWithVersion.value.baseMappingName, closestMaterializedWithVersion.version, desiredVersion, @@ -292,31 +286,31 @@ class EditableMappingService @Inject()( } yield updated } yield updatedEditableMappingInfo - private def getClosestMaterialized(editableMappingId: String, + private def getClosestMaterialized(tracingId: String, desiredVersion: Long): Fox[VersionedKeyValuePair[EditableMappingInfo]] = - tracingDataStore.editableMappingsInfo.get(editableMappingId, version = Some(desiredVersion))( + tracingDataStore.editableMappingsInfo.get(tracingId, version = Some(desiredVersion))( fromProtoBytes[EditableMappingInfo]) - def getClosestMaterializableVersionOrZero(editableMappingId: String, desiredVersion: Option[Long]): Fox[Long] = - tracingDataStore.editableMappingUpdates.getVersion(editableMappingId, + def getClosestMaterializableVersionOrZero(tracingId: String, desiredVersion: Option[Long]): Fox[Long] = + tracingDataStore.editableMappingUpdates.getVersion(tracingId, version = desiredVersion, mayBeEmpty = Some(true), emptyFallback = Some(0L)) - private def getPendingUpdates(editableMappingId: String, + private def getPendingUpdates(tracingId: String, closestMaterializedVersion: Long, closestMaterializableVersion: Long): Fox[List[UpdateAction]] = if (closestMaterializableVersion == closestMaterializedVersion) { Fox.successful(List.empty) } else { for { - updates <- getUpdateActionsWithVersions(editableMappingId, + updates <- getUpdateActionsWithVersions(tracingId, newestVersion = closestMaterializableVersion, oldestVersion = closestMaterializedVersion + 1L) } yield updates.map(_._2).reverse.flatten } - private def getUpdateActionsWithVersions(editableMappingId: String, + private def getUpdateActionsWithVersions(tracingId: String, newestVersion: Long, oldestVersion: Long): Fox[List[(Long, List[UpdateAction])]] = { val batchRanges = batchRangeInclusive(oldestVersion, newestVersion, batchSize = 100) @@ -326,7 +320,7 @@ class EditableMappingService @Inject()( val batchTo = batchRange._2 for { res <- tracingDataStore.editableMappingUpdates.getMultipleVersionsAsVersionValueTuple[List[UpdateAction]]( - editableMappingId, + tracingId, Some(batchTo), Some(batchFrom) )(fromJsonBytes[List[UpdateAction]]) @@ -357,42 +351,42 @@ class EditableMappingService @Inject()( } yield voxelAsLong def volumeData(tracing: VolumeTracing, tracingId: String, dataRequests: DataRequestCollection)( - implicit tc: TokenContext): Fox[(Array[Byte], List[Int])] = - for { - editableMappingId <- tracing.mappingName.toFox - dataLayer = editableMappingLayer(editableMappingId, tracing, tracingId) - requests = dataRequests.map(r => - DataServiceDataRequest(null, dataLayer, r.cuboid(dataLayer), r.settings.copy(appliedAgglomerate = None))) - data <- binaryDataService.handleDataRequests(requests) - } yield data + implicit tc: TokenContext): Fox[(Array[Byte], List[Int])] = { + + val dataLayer = editableMappingLayer(tracingId, tracing, tracingId) + val requests = dataRequests.map(r => + DataServiceDataRequest(null, dataLayer, r.cuboid(dataLayer), r.settings.copy(appliedAgglomerate = None))) + + binaryDataService.handleDataRequests(requests) + } private def getSegmentToAgglomerateForSegmentIds(segmentIds: Set[Long], - editableMappingId: String, + tracingId: String, version: Long): Fox[Map[Long, Long]] = { val chunkIds = segmentIds.map(_ / defaultSegmentToAgglomerateChunkSize) for { maps: List[Seq[(Long, Long)]] <- Fox.serialCombined(chunkIds.toList)(chunkId => - getSegmentToAgglomerateChunkFiltered(editableMappingId, chunkId, version, segmentIds)) + getSegmentToAgglomerateChunkFiltered(tracingId, chunkId, version, segmentIds)) } yield maps.flatten.toMap } - private def getSegmentToAgglomerateChunkFiltered(editableMappingId: String, + private def getSegmentToAgglomerateChunkFiltered(tracingId: String, chunkId: Long, version: Long, segmentIds: Set[Long]): Fox[Seq[(Long, Long)]] = for { - segmentToAgglomerateChunk <- getSegmentToAgglomerateChunkWithEmptyFallback(editableMappingId, chunkId, version) + segmentToAgglomerateChunk <- getSegmentToAgglomerateChunkWithEmptyFallback(tracingId, chunkId, version) filtered = segmentToAgglomerateChunk.filter(pair => segmentIds.contains(pair._1)) } yield filtered - def getSegmentToAgglomerateChunkWithEmptyFallback(editableMappingId: String, + def getSegmentToAgglomerateChunkWithEmptyFallback(tracingId: String, chunkId: Long, version: Long): Fox[Seq[(Long, Long)]] = segmentToAgglomerateChunkCache.getOrLoad( - (editableMappingId, chunkId, version), + (tracingId, chunkId, version), _ => for { - chunkBox: Box[Seq[(Long, Long)]] <- getSegmentToAgglomerateChunk(editableMappingId, chunkId, Some(version)).futureBox + chunkBox: Box[Seq[(Long, Long)]] <- getSegmentToAgglomerateChunk(tracingId, chunkId, Some(version)).futureBox segmentToAgglomerate <- chunkBox match { case Full(chunk) => Fox.successful(chunk) case Empty => Fox.successful(Seq.empty[(Long, Long)]) @@ -401,10 +395,10 @@ class EditableMappingService @Inject()( } yield segmentToAgglomerate ) - private def getSegmentToAgglomerateChunk(editableMappingId: String, + private def getSegmentToAgglomerateChunk(tracingId: String, chunkId: Long, version: Option[Long]): Fox[Seq[(Long, Long)]] = { - val chunkKey = segmentToAgglomerateKey(editableMappingId, chunkId) + val chunkKey = segmentToAgglomerateKey(tracingId, chunkId) getSegmentToAgglomerateChunk(chunkKey, version) } @@ -421,11 +415,11 @@ class EditableMappingService @Inject()( segmentIds: Set[Long], editableMapping: EditableMappingInfo, editableMappingVersion: Long, - editableMappingId: String, + tracingId: String, remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[Map[Long, Long]] = for { editableMappingForSegmentIds <- getSegmentToAgglomerateForSegmentIds(segmentIds, - editableMappingId, + tracingId, editableMappingVersion) segmentIdsInEditableMapping: Set[Long] = editableMappingForSegmentIds.keySet segmentIdsInBaseMapping: Set[Long] = segmentIds.diff(segmentIdsInEditableMapping) @@ -434,17 +428,16 @@ class EditableMappingService @Inject()( remoteFallbackLayer) } yield editableMappingForSegmentIds ++ baseMappingSubset - def getAgglomerateSkeletonWithFallback(editableMappingId: String, + def getAgglomerateSkeletonWithFallback(tracingId: String, remoteFallbackLayer: RemoteFallbackLayer, agglomerateId: Long)(implicit tc: TokenContext): Fox[Array[Byte]] = for { // called here to ensure updates are applied - editableMappingInfo <- getInfo(editableMappingId, version = None, remoteFallbackLayer) - agglomerateGraphBox <- getAgglomerateGraphForId(editableMappingId, agglomerateId, remoteFallbackLayer).futureBox + editableMappingInfo <- getInfo(tracingId, version = None, remoteFallbackLayer) + agglomerateGraphBox <- getAgglomerateGraphForId(tracingId, agglomerateId, remoteFallbackLayer).futureBox skeletonBytes <- agglomerateGraphBox match { case Full(agglomerateGraph) => - Fox.successful( - agglomerateGraphToSkeleton(editableMappingId, agglomerateGraph, remoteFallbackLayer, agglomerateId)) + Fox.successful(agglomerateGraphToSkeleton(tracingId, agglomerateGraph, remoteFallbackLayer, agglomerateId)) case Empty => remoteDatastoreClient.getAgglomerateSkeleton(remoteFallbackLayer, editableMappingInfo.baseMappingName, @@ -453,7 +446,7 @@ class EditableMappingService @Inject()( } } yield skeletonBytes - private def agglomerateGraphToSkeleton(editableMappingId: String, + private def agglomerateGraphToSkeleton(tracingId: String, graph: AgglomerateGraph, remoteFallbackLayer: RemoteFallbackLayer, agglomerateId: Long): Array[Byte] = { @@ -477,7 +470,7 @@ class EditableMappingService @Inject()( createdTimestamp = System.currentTimeMillis(), nodes = nodes, edges = skeletonEdges, - name = s"agglomerate $agglomerateId ($editableMappingId)", + name = s"agglomerate $agglomerateId ($tracingId)", `type` = Some(TreeTypeProto.AGGLOMERATE) )) @@ -488,12 +481,14 @@ class EditableMappingService @Inject()( skeleton.toByteArray } - def getBaseSegmentToAgglomerate(mappingName: String, segmentIds: Set[Long], remoteFallbackLayer: RemoteFallbackLayer)( - implicit tc: TokenContext): Fox[Map[Long, Long]] = { + def getBaseSegmentToAgglomerate( + baseMappingName: String, + segmentIds: Set[Long], + remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[Map[Long, Long]] = { val segmentIdsOrdered = segmentIds.toList for { agglomerateIdsOrdered <- remoteDatastoreClient.getAgglomerateIdsForSegmentIds(remoteFallbackLayer, - mappingName, + baseMappingName, segmentIdsOrdered) } yield segmentIdsOrdered.zip(agglomerateIdsOrdered).toMap } @@ -585,12 +580,12 @@ class EditableMappingService @Inject()( def getAgglomerateGraphForIdWithFallback( mapping: EditableMappingInfo, - editableMappingId: String, + tracingId: String, version: Option[Long], agglomerateId: Long, remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[AgglomerateGraph] = for { - agglomerateGraphBox <- getAgglomerateGraphForId(editableMappingId, agglomerateId, remoteFallbackLayer, version).futureBox + agglomerateGraphBox <- getAgglomerateGraphForId(tracingId, agglomerateId, remoteFallbackLayer, version).futureBox agglomerateGraph <- agglomerateGraphBox match { case Full(agglomerateGraph) => Fox.successful(agglomerateGraph) case Empty => @@ -599,13 +594,13 @@ class EditableMappingService @Inject()( } } yield agglomerateGraph - def agglomerateGraphMinCut(parameters: MinCutParameters, remoteFallbackLayer: RemoteFallbackLayer)( + def agglomerateGraphMinCut(tracingId: String, parameters: MinCutParameters, remoteFallbackLayer: RemoteFallbackLayer)( implicit tc: TokenContext): Fox[List[EdgeWithPositions]] = for { // called here to ensure updates are applied - mapping <- getInfo(parameters.editableMappingId, version = None, remoteFallbackLayer) + mapping <- getInfo(tracingId, version = None, remoteFallbackLayer) agglomerateGraph <- getAgglomerateGraphForIdWithFallback(mapping, - parameters.editableMappingId, + tracingId, None, parameters.agglomerateId, remoteFallbackLayer) @@ -665,13 +660,15 @@ class EditableMappingService @Inject()( ) } - def agglomerateGraphNeighbors(parameters: NeighborsParameters, remoteFallbackLayer: RemoteFallbackLayer)( - implicit tc: TokenContext): Fox[(Long, Seq[NodeWithPosition])] = + def agglomerateGraphNeighbors( + tracingId: String, + parameters: NeighborsParameters, + remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[(Long, Seq[NodeWithPosition])] = for { // called here to ensure updates are applied - mapping <- getInfo(parameters.editableMappingId, version = None, remoteFallbackLayer) + mapping <- getInfo(tracingId, version = None, remoteFallbackLayer) agglomerateGraph <- getAgglomerateGraphForIdWithFallback(mapping, - parameters.editableMappingId, + tracingId, None, parameters.agglomerateId, remoteFallbackLayer) @@ -690,15 +687,14 @@ class EditableMappingService @Inject()( } def merge(newTracingId: String, tracingIds: List[String], remoteFallbackLayer: RemoteFallbackLayer)( - implicit tc: TokenContext): Fox[String] = + implicit tc: TokenContext): Fox[Unit] = for { - firstMappingId <- tracingIds.headOption.toFox + firstTracingId <- tracingIds.headOption.toFox before = Instant.now - _ <- duplicate(Some(firstMappingId), newTracingId, version = None, Some(remoteFallbackLayer)) - _ <- Fox.serialCombined(tracingIds.tail)(editableMappingId => - mergeInto(newTracingId, editableMappingId, remoteFallbackLayer)) + _ <- duplicate(firstTracingId, newTracingId, version = None, Some(remoteFallbackLayer)) + _ <- Fox.serialCombined(tracingIds.tail)(tracingId => mergeInto(newTracingId, tracingId, remoteFallbackLayer)) _ = logger.info(s"Merging ${tracingIds.length} editable mappings took ${Instant.since(before)}") - } yield newTracingId + } yield () // read as: merge source into target (mutate target) private def mergeInto(targetTracingId: String, sourceTracingId: String, remoteFallbackLayer: RemoteFallbackLayer)( diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index c2a281f39f9..d0baa685453 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -32,7 +32,7 @@ import scala.jdk.CollectionConverters.CollectionHasAsScala // this results in only one version increment in the db per update group class EditableMappingUpdater( - editableMappingId: String, + tracingId: String, baseMappingName: String, oldVersion: Long, newVersion: Long, @@ -67,7 +67,7 @@ class EditableMappingUpdater( for { _ <- Fox.serialCombined(segmentToAgglomerateBuffer.keys.toList)(flushSegmentToAgglomerateChunk) _ <- Fox.serialCombined(agglomerateToGraphBuffer.keys.toList)(flushAgglomerateGraph) - _ <- tracingDataStore.editableMappingsInfo.put(editableMappingId, newVersion, updatedEditableMappingInfo) + _ <- tracingDataStore.editableMappingsInfo.put(tracingId, newVersion, updatedEditableMappingInfo) } yield () private def flushSegmentToAgglomerateChunk(key: String): Fox[Unit] = { @@ -137,10 +137,10 @@ class EditableMappingUpdater( agglomerateGraph <- agglomerateGraphForIdWithFallback(editableMappingInfo, agglomerateId) _ = if (segmentId1 == 0) logger.warn( - s"Split action for editable mapping $editableMappingId: Looking up segment id at position ${update.segmentPosition1} in mag ${update.mag} returned invalid value zero. Splitting outside of dataset?") + s"Split action for editable mapping $tracingId: Looking up segment id at position ${update.segmentPosition1} in mag ${update.mag} returned invalid value zero. Splitting outside of dataset?") _ = if (segmentId2 == 0) logger.warn( - s"Split action for editable mapping $editableMappingId: Looking up segment id at position ${update.segmentPosition2} in mag ${update.mag} returned invalid value zero. Splitting outside of dataset?") + s"Split action for editable mapping $tracingId: Looking up segment id at position ${update.segmentPosition2} in mag ${update.mag} returned invalid value zero. Splitting outside of dataset?") (graph1, graph2) <- tryo(splitGraph(agglomerateId, agglomerateGraph, update, segmentId1, segmentId2)) ?~> s"splitGraph failed while removing edge between segments $segmentId1 and $segmentId2" largestExistingAgglomerateId <- largestAgglomerateId(editableMappingInfo) agglomerateId2 = largestExistingAgglomerateId + 1L @@ -183,12 +183,12 @@ class EditableMappingUpdater( private def agglomerateIdForSegmentId(segmentId: Long)(implicit ec: ExecutionContext): Fox[Long] = { val chunkId = segmentId / editableMappingService.defaultSegmentToAgglomerateChunkSize - val chunkKey = segmentToAgglomerateKey(editableMappingId, chunkId) + val chunkKey = segmentToAgglomerateKey(tracingId, chunkId) val chunkFromBufferOpt = getFromSegmentToAgglomerateBuffer(chunkKey) for { chunk <- Fox.fillOption(chunkFromBufferOpt) { editableMappingService - .getSegmentToAgglomerateChunkWithEmptyFallback(editableMappingId, chunkId, version = oldVersion) + .getSegmentToAgglomerateChunkWithEmptyFallback(tracingId, chunkId, version = oldVersion) .map(_.toMap) } agglomerateId <- chunk.get(segmentId) match { @@ -214,38 +214,35 @@ class EditableMappingUpdater( private def updateSegmentToAgglomerateChunk(agglomerateId: Long, chunkId: Long, segmentIdsToUpdate: Seq[Long])( implicit ec: ExecutionContext): Fox[Unit] = for { - existingChunk: Map[Long, Long] <- getSegmentToAgglomerateChunkWithEmptyFallback(editableMappingId, chunkId) ?~> "failed to get old segment to agglomerate chunk for updating it" + existingChunk: Map[Long, Long] <- getSegmentToAgglomerateChunkWithEmptyFallback(tracingId, chunkId) ?~> "failed to get old segment to agglomerate chunk for updating it" mergedMap = existingChunk ++ segmentIdsToUpdate.map(_ -> agglomerateId).toMap - _ = segmentToAgglomerateBuffer.put(segmentToAgglomerateKey(editableMappingId, chunkId), (mergedMap, false)) + _ = segmentToAgglomerateBuffer.put(segmentToAgglomerateKey(tracingId, chunkId), (mergedMap, false)) } yield () - private def getSegmentToAgglomerateChunkWithEmptyFallback(editableMappingId: String, chunkId: Long)( + private def getSegmentToAgglomerateChunkWithEmptyFallback(tracingId: String, chunkId: Long)( implicit ec: ExecutionContext): Fox[Map[Long, Long]] = { - val key = segmentToAgglomerateKey(editableMappingId, chunkId) + val key = segmentToAgglomerateKey(tracingId, chunkId) val fromBufferOpt = getFromSegmentToAgglomerateBuffer(key) Fox.fillOption(fromBufferOpt) { editableMappingService - .getSegmentToAgglomerateChunkWithEmptyFallback(editableMappingId, chunkId, version = oldVersion) + .getSegmentToAgglomerateChunkWithEmptyFallback(tracingId, chunkId, version = oldVersion) .map(_.toMap) } } private def agglomerateGraphForIdWithFallback(mapping: EditableMappingInfo, agglomerateId: Long)( implicit ec: ExecutionContext): Fox[AgglomerateGraph] = { - val key = agglomerateGraphKey(editableMappingId, agglomerateId) + val key = agglomerateGraphKey(tracingId, agglomerateId) val fromBufferOpt = getFromAgglomerateToGraphBuffer(key) fromBufferOpt.map(Fox.successful(_)).getOrElse { - editableMappingService.getAgglomerateGraphForIdWithFallback(mapping, - editableMappingId, - Some(oldVersion), - agglomerateId, - remoteFallbackLayer, - )(tokenContext) + editableMappingService + .getAgglomerateGraphForIdWithFallback(mapping, tracingId, Some(oldVersion), agglomerateId, remoteFallbackLayer, + )(tokenContext) } } private def updateAgglomerateGraph(agglomerateId: Long, graph: AgglomerateGraph): Unit = { - val key = agglomerateGraphKey(editableMappingId, agglomerateId) + val key = agglomerateGraphKey(tracingId, agglomerateId) agglomerateToGraphBuffer.put(key, (graph, false)) } @@ -264,7 +261,7 @@ class EditableMappingUpdater( if (edgesAndAffinitiesMinusOne.length == agglomerateGraph.edges.length) { if (relyOnAgglomerateIds) { logger.warn( - s"Split action for editable mapping $editableMappingId: Edge to remove ($segmentId1 at ${update.segmentPosition1} in mag ${update.mag} to $segmentId2 at ${update.segmentPosition2} in mag ${update.mag} in agglomerate $agglomerateId) already absent. This split becomes a no-op.") + s"Split action for editable mapping $tracingId: Edge to remove ($segmentId1 at ${update.segmentPosition1} in mag ${update.mag} to $segmentId2 at ${update.segmentPosition2} in mag ${update.mag} in agglomerate $agglomerateId) already absent. This split becomes a no-op.") } (agglomerateGraph, emptyAgglomerateGraph) } else { @@ -351,10 +348,10 @@ class EditableMappingUpdater( update.mag)(tokenContext) _ = if (segmentId1 == 0) logger.warn( - s"Merge action for editable mapping $editableMappingId: Looking up segment id at position ${update.segmentPosition1} in mag ${update.mag} returned invalid value zero. Merging outside of dataset?") + s"Merge action for editable mapping $tracingId: Looking up segment id at position ${update.segmentPosition1} in mag ${update.mag} returned invalid value zero. Merging outside of dataset?") _ = if (segmentId2 == 0) logger.warn( - s"Merge action for editable mapping $editableMappingId: Looking up segment id at position ${update.segmentPosition2} in mag ${update.mag} returned invalid value zero. Merging outside of dataset?") + s"Merge action for editable mapping $tracingId: Looking up segment id at position ${update.segmentPosition2} in mag ${update.mag} returned invalid value zero. Merging outside of dataset?") (agglomerateId1, agglomerateId2) <- agglomerateIdsForMergeAction(update, segmentId1, segmentId2) ?~> "Failed to look up agglomerate ids for merge action segments" agglomerateGraph1 <- agglomerateGraphForIdWithFallback(mapping, agglomerateId1) ?~> s"Failed to get agglomerate graph for id $agglomerateId1" agglomerateGraph2 <- agglomerateGraphForIdWithFallback(mapping, agglomerateId2) ?~> s"Failed to get agglomerate graph for id $agglomerateId2" @@ -406,7 +403,7 @@ class EditableMappingUpdater( agglomerateId: Long): Unit = if (!isValid && relyOnAgglomerateIds) { logger.warn( - s"Merge action for editable mapping $editableMappingId: segment $segmentId as looked up at $position in mag $mag is not present in agglomerate $agglomerateId. This merge becomes a no-op" + s"Merge action for editable mapping $tracingId: segment $segmentId as looked up at $position in mag $mag is not present in agglomerate $agglomerateId. This merge becomes a no-op" ) } @@ -414,13 +411,12 @@ class EditableMappingUpdater( implicit ec: ExecutionContext): Fox[EditableMappingInfo] = for { _ <- bool2Fox(revertAction.sourceVersion <= oldVersion) ?~> "trying to revert editable mapping to a version not yet present in the database" - oldInfo <- editableMappingService.getInfo(editableMappingId, - Some(revertAction.sourceVersion), - remoteFallbackLayer)(tokenContext) + oldInfo <- editableMappingService.getInfo(tracingId, Some(revertAction.sourceVersion), remoteFallbackLayer)( + tokenContext) _ = segmentToAgglomerateBuffer.clear() _ = agglomerateToGraphBuffer.clear() segmentToAgglomerateChunkNewestStream = new VersionedSegmentToAgglomerateChunkIterator( - editableMappingId, + tracingId, tracingDataStore.editableMappingsSegmentToAgglomerate) _ <- Fox.serialCombined(segmentToAgglomerateChunkNewestStream) { case (chunkKey, _, version) => @@ -437,7 +433,7 @@ class EditableMappingUpdater( } else Fox.successful(()) } agglomerateToGraphNewestStream = new VersionedAgglomerateToGraphIterator( - editableMappingId, + tracingId, tracingDataStore.editableMappingsAgglomerateToGraph) _ <- Fox.serialCombined(agglomerateToGraphNewestStream) { case (graphKey, _, version) => @@ -445,7 +441,7 @@ class EditableMappingUpdater( for { agglomerateId <- agglomerateIdFromAgglomerateGraphKey(graphKey) _ <- editableMappingService - .getAgglomerateGraphForId(editableMappingId, + .getAgglomerateGraphForId(tracingId, agglomerateId, remoteFallbackLayer, Some(revertAction.sourceVersion))(tokenContext) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala index ad512b77dd7..d11963ca32a 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala @@ -134,6 +134,6 @@ class SkeletonTracingService @Inject()( def dummyTracing: SkeletonTracing = SkeletonTracingDefaults.createInstance def mergeEditableMappings(newTracingId: String, tracingsWithIds: List[(SkeletonTracing, String)])( - implicit tc: TokenContext): Fox[String] = + implicit tc: TokenContext): Fox[Unit] = Fox.empty } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 4882e4fc21e..69ff4ec495d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -23,11 +23,7 @@ import com.scalableminds.webknossos.datastore.models.{ WebknossosAdHocMeshRequest } import com.scalableminds.webknossos.datastore.services._ -import com.scalableminds.webknossos.tracingstore.annotation.{ - AnnotationWithTracings, - TSAnnotationService, - UpdateActionGroup -} +import com.scalableminds.webknossos.tracingstore.annotation.{TSAnnotationService, UpdateActionGroup} import com.scalableminds.webknossos.tracingstore.tracings.TracingType.TracingType import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService @@ -995,17 +991,15 @@ class VolumeTracingService @Inject()( def dummyTracing: VolumeTracing = ??? def mergeEditableMappings(newTracingId: String, tracingsWithIds: List[(VolumeTracing, String)])( - implicit tc: TokenContext): Fox[String] = + implicit tc: TokenContext): Fox[Unit] = if (tracingsWithIds.forall(tracingWithId => tracingWithId._1.getHasEditableMapping)) { for { remoteFallbackLayers <- Fox.serialCombined(tracingsWithIds)(tracingWithId => remoteFallbackLayerFromVolumeTracing(tracingWithId._1, tracingWithId._2)) remoteFallbackLayer <- remoteFallbackLayers.headOption.toFox _ <- bool2Fox(remoteFallbackLayers.forall(_ == remoteFallbackLayer)) ?~> "Cannot merge editable mappings based on different dataset layers" - editableMappingIds <- Fox.serialCombined(tracingsWithIds)(tracingWithId => tracingWithId._1.mappingName) - _ <- bool2Fox(editableMappingIds.length == tracingsWithIds.length) ?~> "Not all volume tracings have editable mappings" - newEditableMappingId <- editableMappingService.merge(newTracingId, editableMappingIds, remoteFallbackLayer) - } yield newEditableMappingId + _ <- editableMappingService.merge(newTracingId, tracingsWithIds.map(_._2), remoteFallbackLayer) + } yield () } else if (tracingsWithIds.forall(tracingWithId => !tracingWithId._1.getHasEditableMapping)) { Fox.empty } else { From 0044b29bb84f33811ff571656b40969ca5f6502e Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 18 Sep 2024 10:47:00 +0200 Subject: [PATCH 063/150] editableMappingController --- .../EditableMappingController.scala | 204 ++++++++++++++++++ .../controllers/VolumeTracingController.scala | 190 +--------------- ...alableminds.webknossos.tracingstore.routes | 16 +- 3 files changed, 217 insertions(+), 193 deletions(-) create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala new file mode 100644 index 00000000000..f514fc43e19 --- /dev/null +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala @@ -0,0 +1,204 @@ +package com.scalableminds.webknossos.tracingstore.controllers + +import com.google.inject.Inject +import com.scalableminds.util.tools.Fox +import com.scalableminds.webknossos.datastore.AgglomerateGraph.AgglomerateGraph +import com.scalableminds.webknossos.datastore.ListOfLong.ListOfLong +import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing +import com.scalableminds.webknossos.datastore.controllers.Controller +import com.scalableminds.webknossos.datastore.services.{ + AccessTokenService, + EditableMappingSegmentListResult, + UserAccessRequest +} +import com.scalableminds.webknossos.tracingstore.annotation.{AnnotationTransactionService, UpdateActionGroup} +import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ + EditableMappingService, + MinCutParameters, + NeighborsParameters +} +import com.scalableminds.webknossos.tracingstore.tracings.volume.{UpdateMappingNameVolumeAction, VolumeTracingService} +import net.liftweb.common.{Box, Empty, Failure, Full} +import play.api.libs.json.Json +import play.api.mvc.{Action, AnyContent, PlayBodyParsers} + +import scala.concurrent.ExecutionContext + +class EditableMappingController @Inject()(volumeTracingService: VolumeTracingService, + accessTokenService: AccessTokenService, + editableMappingService: EditableMappingService, + annotationTransactionService: AnnotationTransactionService)( + implicit ec: ExecutionContext, + bodyParsers: PlayBodyParsers) + extends Controller { + + def makeMappingEditable(token: Option[String], annotationId: String, tracingId: String): Action[AnyContent] = + Action.async { implicit request => + log() { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + for { + tracing <- volumeTracingService.find(annotationId, tracingId) + tracingMappingName <- tracing.mappingName ?~> "annotation.noMappingSet" + _ <- assertMappingIsNotLocked(tracing) + _ <- bool2Fox(volumeTracingService.volumeBucketsAreEmpty(tracingId)) ?~> "annotation.volumeBucketsNotEmpty" + editableMappingInfo <- editableMappingService.create(tracingId, baseMappingName = tracingMappingName) + volumeUpdate = UpdateMappingNameVolumeAction(Some(tracingId), + isEditable = Some(true), + isLocked = Some(true), + actionTracingId = tracingId, + actionTimestamp = Some(System.currentTimeMillis())) + _ <- annotationTransactionService + .handleUpdateGroups( // TODO replace this route by the update action only? address editable mappings by volume tracing id? + annotationId, + List( + UpdateActionGroup(tracing.version + 1, + System.currentTimeMillis(), + None, + List(volumeUpdate), + None, + None, + "dummyTransactionId", + 1, + 0)) + ) + infoJson <- editableMappingService.infoJson(tracingId = tracingId, + editableMappingInfo = editableMappingInfo, + version = Some(0L)) + } yield Ok(infoJson) + } + } + } + + private def assertMappingIsNotLocked(volumeTracing: VolumeTracing): Fox[Unit] = + bool2Fox(!volumeTracing.mappingIsLocked.getOrElse(false)) ?~> "annotation.mappingIsLocked" + + /*// TODO integrate all of this into annotation update + + def updateEditableMapping(token: Option[String], + annotationId: String, + tracingId: String): Action[List[UpdateActionGroup]] = + Action.async(validateJson[List[UpdateActionGroup]]) { implicit request => + accessTokenService.validateAccess(UserAccessRequest.writeTracing(tracingId)) { + for { + tracing <- tracingService.find(annotationId, tracingId) + mappingName <- tracing.mappingName.toFox + _ <- editableMappingService.assertTracingHasEditableMapping(tracing) + currentVersion <- editableMappingService.getClosestMaterializableVersionOrZero(mappingName, None) + _ <- bool2Fox(request.body.length == 1) ?~> "Editable mapping update request must contain exactly one update group" + updateGroup <- request.body.headOption.toFox + _ <- bool2Fox(updateGroup.version == currentVersion + 1) ?~> "version mismatch" + report = TracingUpdatesReport( + annotationId, // TODO integrate all of this into annotation update + timestamps = List(Instant(updateGroup.timestamp)), + statistics = None, + significantChangesCount = updateGroup.actions.length, + viewChangesCount = 0, + tokenContextForRequest.userTokenOpt + ) + _ <- remoteWebknossosClient.reportTracingUpdates(report) + remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) + _ <- editableMappingService.update(mappingName, updateGroup, updateGroup.version, remoteFallbackLayer) + } yield Ok + } + } + */ + + def editableMappingInfo(token: Option[String], + annotationId: String, + tracingId: String, + version: Option[Long]): Action[AnyContent] = + Action.async { implicit request => + log() { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + for { + tracing <- volumeTracingService.find(annotationId, tracingId) + _ <- editableMappingService.assertTracingHasEditableMapping(tracing) + editableMappingInfo <- editableMappingService.getInfoNEW(annotationId, tracingId, version) + infoJson <- editableMappingService.infoJson(tracingId = tracingId, + editableMappingInfo = editableMappingInfo, + version = version) + } yield Ok(infoJson) + } + } + } + + def segmentIdsForAgglomerate(token: Option[String], + annotationId: String, + tracingId: String, + agglomerateId: Long): Action[AnyContent] = Action.async { implicit request => + log() { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + for { + tracing <- volumeTracingService.find(annotationId, tracingId) + _ <- editableMappingService.assertTracingHasEditableMapping(tracing) + remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) + agglomerateGraphBox: Box[AgglomerateGraph] <- editableMappingService + .getAgglomerateGraphForId(tracingId, agglomerateId, remoteFallbackLayer) + .futureBox + segmentIds <- agglomerateGraphBox match { + case Full(agglomerateGraph) => Fox.successful(agglomerateGraph.segments) + case Empty => Fox.successful(List.empty) + case f: Failure => f.toFox + } + agglomerateIdIsPresent = agglomerateGraphBox.isDefined + } yield Ok(Json.toJson(EditableMappingSegmentListResult(segmentIds.toList, agglomerateIdIsPresent))) + } + } + } + + def agglomerateIdsForSegments(token: Option[String], annotationId: String, tracingId: String): Action[ListOfLong] = + Action.async(validateProto[ListOfLong]) { implicit request => + log() { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + for { + tracing <- volumeTracingService.find(annotationId, tracingId) + _ <- editableMappingService.assertTracingHasEditableMapping(tracing) + remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) + (editableMappingInfo, editableMappingVersion) <- editableMappingService.getInfoAndActualVersion( + tracingId, + requestedVersion = None, + remoteFallbackLayer = remoteFallbackLayer) + relevantMapping: Map[Long, Long] <- editableMappingService.generateCombinedMappingForSegmentIds( + request.body.items.toSet, + editableMappingInfo, + editableMappingVersion, + tracingId, + remoteFallbackLayer) + agglomerateIdsSorted = relevantMapping.toSeq.sortBy(_._1).map(_._2) + } yield Ok(ListOfLong(agglomerateIdsSorted).toByteArray) + } + } + } + + def agglomerateGraphMinCut(token: Option[String], annotationId: String, tracingId: String): Action[MinCutParameters] = + Action.async(validateJson[MinCutParameters]) { implicit request => + log() { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + for { + tracing <- volumeTracingService.find(annotationId, tracingId) + _ <- editableMappingService.assertTracingHasEditableMapping(tracing) + remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) + edges <- editableMappingService.agglomerateGraphMinCut(tracingId, request.body, remoteFallbackLayer) + } yield Ok(Json.toJson(edges)) + } + } + } + + def agglomerateGraphNeighbors(token: Option[String], + annotationId: String, + tracingId: String): Action[NeighborsParameters] = + Action.async(validateJson[NeighborsParameters]) { implicit request => + log() { + accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + for { + tracing <- volumeTracingService.find(annotationId, tracingId) + _ <- editableMappingService.assertTracingHasEditableMapping(tracing) + remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) + (segmentId, edges) <- editableMappingService.agglomerateGraphNeighbors(tracingId, + request.body, + remoteFallbackLayer) + } yield Ok(Json.obj("segmentId" -> segmentId, "neighbors" -> Json.toJson(edges))) + } + } + } +} diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 3270a11a992..48e73e75094 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -5,8 +5,6 @@ import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int} import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.ExtendedTypes.ExtendedString import com.scalableminds.util.tools.Fox -import com.scalableminds.webknossos.datastore.AgglomerateGraph.AgglomerateGraph -import com.scalableminds.webknossos.datastore.ListOfLong.ListOfLong import com.scalableminds.webknossos.datastore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings} import com.scalableminds.webknossos.datastore.geometry.ListOfVec3IntProto import com.scalableminds.webknossos.datastore.helpers.{ @@ -22,23 +20,14 @@ import com.scalableminds.webknossos.datastore.models.{ WebknossosDataRequest } import com.scalableminds.webknossos.datastore.rpc.RPC -import com.scalableminds.webknossos.datastore.services.{ - EditableMappingSegmentListResult, - FullMeshRequest, - UserAccessRequest -} -import com.scalableminds.webknossos.tracingstore.annotation.{AnnotationTransactionService, UpdateActionGroup} +import com.scalableminds.webknossos.datastore.services.{FullMeshRequest, UserAccessRequest} +import com.scalableminds.webknossos.tracingstore.annotation.AnnotationTransactionService import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService -import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ - EditableMappingService, - MinCutParameters, - NeighborsParameters -} +import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService import com.scalableminds.webknossos.tracingstore.tracings.volume.{ MergedVolumeStats, ResolutionRestrictions, TSFullMeshService, - UpdateMappingNameVolumeAction, VolumeDataZipFormat, VolumeSegmentIndexService, VolumeSegmentStatisticsService, @@ -50,9 +39,8 @@ import com.scalableminds.webknossos.tracingstore.{ TSRemoteWebknossosClient, TracingStoreAccessTokenService, TracingStoreConfig, - TracingUpdatesReport } -import net.liftweb.common.{Box, Empty, Failure, Full} +import net.liftweb.common.Empty import play.api.i18n.Messages import play.api.libs.Files.TemporaryFile import play.api.libs.json.Json @@ -370,176 +358,6 @@ class VolumeTracingController @Inject()( } } - def makeMappingEditable(token: Option[String], annotationId: String, tracingId: String): Action[AnyContent] = - Action.async { implicit request => - log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { - for { - tracing <- tracingService.find(annotationId, tracingId) - tracingMappingName <- tracing.mappingName ?~> "annotation.noMappingSet" - _ <- assertMappingIsNotLocked(tracing) - _ <- bool2Fox(tracingService.volumeBucketsAreEmpty(tracingId)) ?~> "annotation.volumeBucketsNotEmpty" - editableMappingInfo <- editableMappingService.create(tracingId, baseMappingName = tracingMappingName) - volumeUpdate = UpdateMappingNameVolumeAction(Some(tracingId), - isEditable = Some(true), - isLocked = Some(true), - actionTracingId = tracingId, - actionTimestamp = Some(System.currentTimeMillis())) - _ <- annotationTransactionService - .handleUpdateGroups( // TODO replace this route by the update action only? address editable mappings by volume tracing id? - annotationId, - List( - UpdateActionGroup(tracing.version + 1, - System.currentTimeMillis(), - None, - List(volumeUpdate), - None, - None, - "dummyTransactionId", - 1, - 0)) - ) - infoJson <- editableMappingService.infoJson(tracingId = tracingId, - editableMappingInfo = editableMappingInfo, - version = Some(0L)) - } yield Ok(infoJson) - } - } - } - - private def assertMappingIsNotLocked(volumeTracing: VolumeTracing): Fox[Unit] = - bool2Fox(!volumeTracing.mappingIsLocked.getOrElse(false)) ?~> "annotation.mappingIsLocked" - - def agglomerateGraphMinCut(token: Option[String], annotationId: String, tracingId: String): Action[MinCutParameters] = - Action.async(validateJson[MinCutParameters]) { implicit request => - log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { - for { - tracing <- tracingService.find(annotationId, tracingId) - _ <- bool2Fox(tracing.getHasEditableMapping) ?~> "Mapping is not editable" - remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - edges <- editableMappingService.agglomerateGraphMinCut(tracingId, request.body, remoteFallbackLayer) - } yield Ok(Json.toJson(edges)) - } - } - } - - def agglomerateGraphNeighbors(token: Option[String], - annotationId: String, - tracingId: String): Action[NeighborsParameters] = - Action.async(validateJson[NeighborsParameters]) { implicit request => - log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { - for { - tracing <- tracingService.find(annotationId, tracingId) - _ <- editableMappingService.assertTracingHasEditableMapping(tracing) - remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - (segmentId, edges) <- editableMappingService.agglomerateGraphNeighbors(tracingId, - request.body, - remoteFallbackLayer) - } yield Ok(Json.obj("segmentId" -> segmentId, "neighbors" -> Json.toJson(edges))) - } - } - } - - def updateEditableMapping(token: Option[String], - annotationId: String, - tracingId: String): Action[List[UpdateActionGroup]] = - Action.async(validateJson[List[UpdateActionGroup]]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.writeTracing(tracingId)) { - for { - tracing <- tracingService.find(annotationId, tracingId) - mappingName <- tracing.mappingName.toFox - _ <- editableMappingService.assertTracingHasEditableMapping(tracing) - currentVersion <- editableMappingService.getClosestMaterializableVersionOrZero(mappingName, None) - _ <- bool2Fox(request.body.length == 1) ?~> "Editable mapping update request must contain exactly one update group" - updateGroup <- request.body.headOption.toFox - _ <- bool2Fox(updateGroup.version == currentVersion + 1) ?~> "version mismatch" - report = TracingUpdatesReport( - annotationId, // TODO integrate all of this into annotation update - timestamps = List(Instant(updateGroup.timestamp)), - statistics = None, - significantChangesCount = updateGroup.actions.length, - viewChangesCount = 0, - tokenContextForRequest.userTokenOpt - ) - _ <- remoteWebknossosClient.reportTracingUpdates(report) - remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - _ <- editableMappingService.update(mappingName, updateGroup, updateGroup.version, remoteFallbackLayer) - } yield Ok - } - } - - def editableMappingInfo(token: Option[String], - annotationId: String, - tracingId: String, - version: Option[Long]): Action[AnyContent] = - Action.async { implicit request => - log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { - for { - tracing <- tracingService.find(annotationId, tracingId) - _ <- editableMappingService.assertTracingHasEditableMapping(tracing) - editableMappingInfo <- editableMappingService.getInfoNEW(annotationId, tracingId, version) - infoJson <- editableMappingService.infoJson(tracingId = tracingId, - editableMappingInfo = editableMappingInfo, - version = version) - } yield Ok(infoJson) - } - } - } - - def editableMappingAgglomerateIdsForSegments(token: Option[String], - annotationId: String, - tracingId: String): Action[ListOfLong] = - Action.async(validateProto[ListOfLong]) { implicit request => - log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { - for { - tracing <- tracingService.find(annotationId, tracingId) - _ <- editableMappingService.assertTracingHasEditableMapping(tracing) - remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - (editableMappingInfo, editableMappingVersion) <- editableMappingService.getInfoAndActualVersion( - tracingId, - requestedVersion = None, - remoteFallbackLayer = remoteFallbackLayer) - relevantMapping: Map[Long, Long] <- editableMappingService.generateCombinedMappingForSegmentIds( - request.body.items.toSet, - editableMappingInfo, - editableMappingVersion, - tracingId, - remoteFallbackLayer) - agglomerateIdsSorted = relevantMapping.toSeq.sortBy(_._1).map(_._2) - } yield Ok(ListOfLong(agglomerateIdsSorted).toByteArray) - } - } - } - - def editableMappingSegmentIdsForAgglomerate(token: Option[String], - annotationId: String, - tracingId: String, - agglomerateId: Long): Action[AnyContent] = Action.async { - implicit request => - log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { - for { - tracing <- tracingService.find(annotationId, tracingId) - mappingName <- tracing.mappingName.toFox - remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - agglomerateGraphBox: Box[AgglomerateGraph] <- editableMappingService - .getAgglomerateGraphForId(mappingName, agglomerateId, remoteFallbackLayer) - .futureBox - segmentIds <- agglomerateGraphBox match { - case Full(agglomerateGraph) => Fox.successful(agglomerateGraph.segments) - case Empty => Fox.successful(List.empty) - case f: Failure => f.toFox - } - agglomerateIdIsPresent = agglomerateGraphBox.isDefined - } yield Ok(Json.toJson(EditableMappingSegmentListResult(segmentIds.toList, agglomerateIdIsPresent))) - } - } - } - def getSegmentVolume(token: Option[String], annotationId: String, tracingId: String): Action[SegmentStatisticsParameters] = diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 9e9a44a8253..f9a1e371dcd 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -28,9 +28,6 @@ POST /volume/:annotationId/:tracingId/importVolumeData @c POST /volume/:annotationId/:tracingId/addSegmentIndex @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.addSegmentIndex(token: Option[String], annotationId: String, tracingId: String, dryRun: Boolean) GET /volume/:annotationId/:tracingId/findData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.findData(token: Option[String], annotationId: String, tracingId: String) GET /volume/:annotationId/:tracingId/agglomerateSkeleton/:agglomerateId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.agglomerateSkeleton(token: Option[String], annotationId: String, tracingId: String, agglomerateId: Long) -POST /volume/:annotationId/:tracingId/makeMappingEditable @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.makeMappingEditable(token: Option[String], annotationId: String, tracingId: String) -POST /volume/:annotationId/:tracingId/agglomerateGraphMinCut @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.agglomerateGraphMinCut(token: Option[String], annotationId: String, tracingId: String) -POST /volume/:annotationId/:tracingId/agglomerateGraphNeighbors @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.agglomerateGraphNeighbors(token: Option[String], annotationId: String, tracingId: String) POST /volume/:annotationId/:tracingId/segmentStatistics/volume @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentVolume(token: Option[String], annotationId: String, tracingId: String) POST /volume/:annotationId/:tracingId/segmentStatistics/boundingBox @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentBoundingBox(token: Option[String], annotationId: String, tracingId: String) POST /volume/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getMultiple(token: Option[String]) @@ -38,10 +35,15 @@ POST /volume/mergedFromIds @c POST /volume/mergedFromContents @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromContents(token: Option[String], persist: Boolean) # Editable Mappings -POST /mapping/:annotationId/:tracingId/update @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.updateEditableMapping(token: Option[String], annotationId: String, tracingId: String) -GET /mapping/:annotationId/:tracingId/info @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingInfo(token: Option[String], annotationId: String, tracingId: String, version: Option[Long]) -GET /mapping/:annotationId/:tracingId/segmentsForAgglomerate @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingSegmentIdsForAgglomerate(token: Option[String], annotationId: String, tracingId: String, agglomerateId: Long) -POST /mapping/:annotationId/:tracingId/agglomeratesForSegments @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.editableMappingAgglomerateIdsForSegments(token: Option[String], annotationId: String, tracingId: String) +# todo adapt frontend to mapping route prefix +POST /mapping/:annotationId/:tracingId/makeMappingEditable @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.makeMappingEditable(token: Option[String], annotationId: String, tracingId: String) +GET /mapping/:annotationId/:tracingId/info @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.editableMappingInfo(token: Option[String], annotationId: String, tracingId: String, version: Option[Long]) +GET /mapping/:annotationId/:tracingId/segmentsForAgglomerate @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.segmentIdsForAgglomerate(token: Option[String], annotationId: String, tracingId: String, agglomerateId: Long) +POST /mapping/:annotationId/:tracingId/agglomeratesForSegments @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateIdsForSegments(token: Option[String], annotationId: String, tracingId: String) +# todo adapt frontend to mapping route prefix +POST /mapping/:annotationId/:tracingId/agglomerateGraphMinCut @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateGraphMinCut(token: Option[String], annotationId: String, tracingId: String) +# todo adapt frontend to mapping route prefix +POST /mapping/:annotationId/:tracingId/agglomerateGraphNeighbors @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateGraphNeighbors(token: Option[String], annotationId: String, tracingId: String) # Zarr endpoints for volume annotations # Zarr version 2 From d705aeb46773dd692a5f9828f032a0de6250aff7 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 18 Sep 2024 11:08:56 +0200 Subject: [PATCH 064/150] editable mapping info does not need version, mappingName anymore. remove dep from EditableMappingService to TSAnnotationService --- frontend/javascripts/admin/admin_rest_api.ts | 4 ++- .../oxalis/model/reducers/save_reducer.ts | 4 +-- .../oxalis/model/sagas/proofread_saga.ts | 11 +++++-- .../oxalis/model/sagas/save_saga.ts | 2 +- .../model/sagas/skeletontracing_saga.ts | 4 +-- .../javascripts/oxalis/view/version_view.tsx | 2 +- frontend/javascripts/types/api_flow_types.ts | 2 -- .../annotation/TSAnnotationService.scala | 8 +++++ .../EditableMappingController.scala | 17 ++++++----- .../controllers/VolumeTracingController.scala | 10 +++---- .../tracings/TracingService.scala | 6 +--- .../EditableMappingService.scala | 29 +++++-------------- .../skeleton/SkeletonTracingService.scala | 5 ---- .../volume/VolumeTracingService.scala | 5 ---- 14 files changed, 47 insertions(+), 62 deletions(-) diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts index ee6816d8845..e341bf573a1 100644 --- a/frontend/javascripts/admin/admin_rest_api.ts +++ b/frontend/javascripts/admin/admin_rest_api.ts @@ -1633,7 +1633,9 @@ export function getEditableMappingInfo( tracingId: string, ): Promise { return doWithToken((token) => - Request.receiveJSON(`${tracingStoreUrl}/tracings/mapping/${annotationId}/${tracingId}/info?token=${token}`), + Request.receiveJSON( + `${tracingStoreUrl}/tracings/mapping/${annotationId}/${tracingId}/info?token=${token}`, + ), ); } diff --git a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts index 26a9f0e19ce..1fab6d3f403 100644 --- a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts +++ b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts @@ -40,9 +40,9 @@ function updateVersion(state: OxalisState, action: SetVersionNumberAction) { version: action.version, }); } else if (action.saveQueueType === "mapping") { - return updateEditableMapping(state, action.tracingId, { + /*return updateEditableMapping(state, action.tracingId, { version: action.version, - }); + });*/ } return state; diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts index 0fc1a9e5d78..e4e911bf1c6 100644 --- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts @@ -276,13 +276,18 @@ function* createEditableMapping(): Saga { const volumeTracingId = upToDateVolumeTracing.tracingId; const layerName = volumeTracingId; - const serverEditableMapping = yield* call(makeMappingEditable, tracingStoreUrl, annotationId, volumeTracingId); + const serverEditableMapping = yield* call( + makeMappingEditable, + tracingStoreUrl, + annotationId, + volumeTracingId, + ); // The server increments the volume tracing's version by 1 when switching the mapping to an editable one yield* put(setVersionNumberAction(upToDateVolumeTracing.version + 1, "volume", volumeTracingId)); - yield* put(setMappingNameAction(layerName, serverEditableMapping.mappingName, "HDF5")); + yield* put(setMappingNameAction(layerName, serverEditableMapping.tracingId, "HDF5")); yield* put(setHasEditableMappingAction()); yield* put(initializeEditableMappingAction(serverEditableMapping)); - return serverEditableMapping.mappingName; + return serverEditableMapping.tracingId; } function* ensureHdf5MappingIsEnabled(layerName: string): Saga { diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts index 508a62a8e99..d94f8851f83 100644 --- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts @@ -166,7 +166,7 @@ export function* sendSaveRequestToServer(): Saga { const saveQueue = sliceAppropriateBatchCount(fullSaveQueue); let compactedSaveQueue = compactSaveQueue(saveQueue); const tracings = yield* select((state) => - _.compact([state.tracing.skeleton, ...state.tracing.volumes, ...state.tracing.mappings]), + _.compact([state.tracing.skeleton, ...state.tracing.volumes]), ); const version = _.max(tracings.map((t) => t.version)) || 0; const annotationId = yield* select((state) => state.tracing.annotationId); diff --git a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts index b83a9531439..91e2d317cdf 100644 --- a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts @@ -250,9 +250,7 @@ function* getAgglomerateSkeletonTracing( const annotation = yield* select((state) => state.tracing); const layerInfo = getLayerByName(dataset, layerName); - const editableMapping = annotation.mappings.find( - (mapping) => mapping.mappingName === mappingName, - ); + const editableMapping = annotation.mappings.find((mapping) => mapping.tracingId === mappingName); try { let nmlProtoBuffer; diff --git a/frontend/javascripts/oxalis/view/version_view.tsx b/frontend/javascripts/oxalis/view/version_view.tsx index 8068ad59be6..4a8862b7719 100644 --- a/frontend/javascripts/oxalis/view/version_view.tsx +++ b/frontend/javascripts/oxalis/view/version_view.tsx @@ -87,7 +87,7 @@ class VersionView extends React.Component { this.props.tracing, mapping.tracingId, )} (Editable Mapping)`, - key: `${mapping.tracingId}-${mapping.mappingName}`, + key: mapping.tracingId, children: ( "applyUpdates.failed" } yield updated + def getEditableMappingInfo(annotationId: String, tracingId: String, version: Option[Long] = None)( + implicit ec: ExecutionContext, + tc: TokenContext): Fox[EditableMappingInfo] = + for { + annotation <- getWithTracings(annotationId, version, List(tracingId), List.empty) + tracing <- annotation.getEditableMappingInfo(tracingId) + } yield tracing + private def applyPendingUpdates(annotation: AnnotationProto, annotationId: String, targetVersionOpt: Option[Long], diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala index f514fc43e19..a8a9484ba11 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala @@ -11,7 +11,11 @@ import com.scalableminds.webknossos.datastore.services.{ EditableMappingSegmentListResult, UserAccessRequest } -import com.scalableminds.webknossos.tracingstore.annotation.{AnnotationTransactionService, UpdateActionGroup} +import com.scalableminds.webknossos.tracingstore.annotation.{ + AnnotationTransactionService, + TSAnnotationService, + UpdateActionGroup +} import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ EditableMappingService, MinCutParameters, @@ -25,6 +29,7 @@ import play.api.mvc.{Action, AnyContent, PlayBodyParsers} import scala.concurrent.ExecutionContext class EditableMappingController @Inject()(volumeTracingService: VolumeTracingService, + annotationService: TSAnnotationService, accessTokenService: AccessTokenService, editableMappingService: EditableMappingService, annotationTransactionService: AnnotationTransactionService)( @@ -61,9 +66,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer 1, 0)) ) - infoJson <- editableMappingService.infoJson(tracingId = tracingId, - editableMappingInfo = editableMappingInfo, - version = Some(0L)) + infoJson = editableMappingService.infoJson(tracingId = tracingId, editableMappingInfo = editableMappingInfo) } yield Ok(infoJson) } } @@ -113,10 +116,8 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer for { tracing <- volumeTracingService.find(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) - editableMappingInfo <- editableMappingService.getInfoNEW(annotationId, tracingId, version) - infoJson <- editableMappingService.infoJson(tracingId = tracingId, - editableMappingInfo = editableMappingInfo, - version = version) + editableMappingInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId, version) + infoJson = editableMappingService.infoJson(tracingId = tracingId, editableMappingInfo = editableMappingInfo) } yield Ok(infoJson) } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 48e73e75094..7d3d5871167 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -21,7 +21,7 @@ import com.scalableminds.webknossos.datastore.models.{ } import com.scalableminds.webknossos.datastore.rpc.RPC import com.scalableminds.webknossos.datastore.services.{FullMeshRequest, UserAccessRequest} -import com.scalableminds.webknossos.tracingstore.annotation.AnnotationTransactionService +import com.scalableminds.webknossos.tracingstore.annotation.TSAnnotationService import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService import com.scalableminds.webknossos.tracingstore.tracings.volume.{ @@ -38,7 +38,7 @@ import com.scalableminds.webknossos.tracingstore.{ TSRemoteDatastoreClient, TSRemoteWebknossosClient, TracingStoreAccessTokenService, - TracingStoreConfig, + TracingStoreConfig } import net.liftweb.common.Empty import play.api.i18n.Messages @@ -55,7 +55,7 @@ class VolumeTracingController @Inject()( val config: TracingStoreConfig, val remoteDataStoreClient: TSRemoteDatastoreClient, val accessTokenService: TracingStoreAccessTokenService, - annotationTransactionService: AnnotationTransactionService, + annotationService: TSAnnotationService, editableMappingService: EditableMappingService, val slackNotificationService: TSSlackNotificationService, val remoteWebknossosClient: TSRemoteWebknossosClient, @@ -261,12 +261,12 @@ class VolumeTracingController @Inject()( accessTokenService.validateAccess(UserAccessRequest.webknossos) { for { tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") - currentVersion <- tracingService.currentVersion(tracingId) + currentVersion <- annotationService.currentMaterializableVersion(tracingId) before = Instant.now canAddSegmentIndex <- tracingService.checkIfSegmentIndexMayBeAdded(tracingId, tracing) processedBucketCountOpt <- Fox.runIf(canAddSegmentIndex)(tracingService .addSegmentIndex(annotationId, tracingId, tracing, currentVersion, dryRun)) ?~> "addSegmentIndex.failed" - currentVersionNew <- tracingService.currentVersion(tracingId) + currentVersionNew <- annotationService.currentMaterializableVersion(tracingId) _ <- Fox.runIf(!dryRun)(bool2Fox( processedBucketCountOpt.isEmpty || currentVersionNew == currentVersion + 1L) ?~> "Version increment failed. Looks like someone edited the annotation layer in the meantime.") duration = Instant.since(before) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala index f6a1d146e0c..ae23b35d8d4 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala @@ -60,10 +60,6 @@ trait TracingService[T <: GeneratedMessage] private val handledGroupCacheExpiry: FiniteDuration = 24 hours - def currentVersion(tracingId: String): Fox[Long] - - def currentVersion(tracing: T): Long - private def transactionGroupKey(tracingId: String, transactionId: String, transactionGroupIndex: Int, version: Long) = s"transactionGroup___${tracingId}___${transactionId}___${transactionGroupIndex}___$version" @@ -125,7 +121,7 @@ trait TracingService[T <: GeneratedMessage] implicit tc: TokenContext): Fox[List[Option[T]]] = Fox.combined { selectors.map { - case Some(selector) => + case Some(selector) => // TODO TracingSelector needs annotationIds too find("dummyAnnotationid", selector.tracingId, selector.version, useCache, applyUpdates).map(Some(_)) case None => Fox.successful(None) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index dcf2a0bbfef..3345d9d94be 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -22,7 +22,7 @@ import com.scalableminds.webknossos.datastore.services.{ AdHocMeshServiceHolder, BinaryDataService } -import com.scalableminds.webknossos.tracingstore.annotation.{TSAnnotationService, UpdateAction, UpdateActionGroup} +import com.scalableminds.webknossos.tracingstore.annotation.{UpdateAction, UpdateActionGroup} import com.scalableminds.webknossos.tracingstore.tracings.volume.ReversionHelper import com.scalableminds.webknossos.tracingstore.tracings.{ FallbackDataHelper, @@ -91,7 +91,6 @@ object NodeWithPosition { class EditableMappingService @Inject()( val tracingDataStore: TracingDataStore, val adHocMeshServiceHolder: AdHocMeshServiceHolder, - annotationService: TSAnnotationService, val remoteDatastoreClient: TSRemoteDatastoreClient, val remoteWebknossosClient: TSRemoteWebknossosClient )(implicit ec: ExecutionContext) @@ -117,18 +116,13 @@ class EditableMappingService @Inject()( private lazy val agglomerateToGraphCache: AlfuCache[(String, Long, Long), AgglomerateGraph] = AlfuCache(maxCapacity = 50) - def infoJson(tracingId: String, editableMappingInfo: EditableMappingInfo, version: Option[Long]): Fox[JsObject] = - for { - version <- getClosestMaterializableVersionOrZero(tracingId, version) - } yield - Json.obj( - "mappingName" -> tracingId, // TODO remove? - "version" -> version, - "tracingId" -> tracingId, - "baseMappingName" -> editableMappingInfo.baseMappingName, - "largestAgglomerateId" -> editableMappingInfo.largestAgglomerateId, - "createdTimestamp" -> editableMappingInfo.createdTimestamp - ) + def infoJson(tracingId: String, editableMappingInfo: EditableMappingInfo): JsObject = + Json.obj( + "tracingId" -> tracingId, + "baseMappingName" -> editableMappingInfo.baseMappingName, + "largestAgglomerateId" -> editableMappingInfo.largestAgglomerateId, + "createdTimestamp" -> editableMappingInfo.createdTimestamp + ) def create(tracingId: String, baseMappingName: String): Fox[EditableMappingInfo] = { val newEditableMappingInfo = EditableMappingInfo( @@ -193,13 +187,6 @@ class EditableMappingService @Inject()( } yield () } - def getInfoNEW(annotationId: String, tracingId: String, version: Option[Long] = None)( - implicit tc: TokenContext): Fox[EditableMappingInfo] = - for { - annotation <- annotationService.getWithTracings(annotationId, version, List(tracingId), List.empty) - tracing <- annotation.getEditableMappingInfo(tracingId) - } yield tracing - def getInfo(tracingId: String, version: Option[Long] = None, remoteFallbackLayer: RemoteFallbackLayer)( implicit tc: TokenContext): Fox[EditableMappingInfo] = for { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala index d11963ca32a..e952eb33f6f 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala @@ -36,11 +36,6 @@ class SkeletonTracingService @Inject()( implicit val tracingCompanion: SkeletonTracing.type = SkeletonTracing - def currentVersion(tracingId: String): Fox[Long] = - tracingDataStore.skeletonUpdates.getVersion(tracingId, mayBeEmpty = Some(true), emptyFallback = Some(0L)) - - def currentVersion(tracing: SkeletonTracing): Long = tracing.version - def find(annotationId: String, tracingId: String, version: Option[Long] = None, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 69ff4ec495d..7d2f76dc0ed 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -96,11 +96,6 @@ class VolumeTracingService @Inject()( private val fallbackLayerCache: AlfuCache[(String, String, Option[String]), Option[RemoteFallbackLayer]] = AlfuCache( maxCapacity = 100) - override def currentVersion(tracingId: String): Fox[Long] = - tracingDataStore.volumes.getVersion(tracingId, mayBeEmpty = Some(true), emptyFallback = Some(0L)) - - override def currentVersion(tracing: VolumeTracing): Long = tracing.version - override protected def updateSegmentIndex( segmentIndexBuffer: VolumeSegmentIndexBuffer, bucketPosition: BucketPosition, From ef0952863e13c2032b3af7ff356bf1d49b5e8d52 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 18 Sep 2024 11:36:20 +0200 Subject: [PATCH 065/150] renamings, construct EditableMappingUpdater in TSAnnotationService --- .../controllers/BinaryDataController.scala | 18 ++--- .../controllers/DSMeshController.scala | 8 +-- .../controllers/DataSourceController.scala | 70 +++++++++---------- .../controllers/ExportsController.scala | 2 +- .../controllers/ZarrStreamingController.scala | 20 +++--- .../services/AccessTokenService.scala | 10 +-- .../annotation/AnnotationWithTracings.scala | 9 ++- .../annotation/TSAnnotationService.scala | 62 +++++++++++++--- .../EditableMappingController.scala | 12 ++-- .../SkeletonTracingController.scala | 4 +- .../controllers/TSAnnotationController.scala | 12 ++-- .../controllers/TracingController.scala | 10 +-- .../controllers/VolumeTracingController.scala | 32 ++++----- ...VolumeTracingZarrStreamingController.scala | 22 +++--- .../volume/VolumeTracingService.scala | 2 + 15 files changed, 171 insertions(+), 122 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala index 50541f438c9..514ea193faa 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala @@ -53,7 +53,7 @@ class BinaryDataController @Inject()( datasetName: String, dataLayerName: String ): Action[List[WebknossosDataRequest]] = Action.async(validateJson[List[WebknossosDataRequest]]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { logTime(slackNotificationService.noticeSlowRequest) { val t = Instant.now for { @@ -95,7 +95,7 @@ class BinaryDataController @Inject()( halfByte: Boolean, mappingName: Option[String] ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -119,7 +119,7 @@ class BinaryDataController @Inject()( datasetName: String, dataLayerName: String ): Action[RawCuboidRequest] = Action.async(validateJson[RawCuboidRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -141,7 +141,7 @@ class BinaryDataController @Inject()( y: Int, z: Int, cubeSize: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -175,7 +175,7 @@ class BinaryDataController @Inject()( intensityMax: Option[Double], color: Option[String], invertColor: Option[Boolean]): Action[RawBuffer] = Action.async(parse.raw) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -222,7 +222,7 @@ class BinaryDataController @Inject()( dataLayerName: String, mappingName: String ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -242,7 +242,7 @@ class BinaryDataController @Inject()( datasetName: String, dataLayerName: String): Action[WebknossosAdHocMeshRequest] = Action.async(validateJson[WebknossosAdHocMeshRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -283,7 +283,7 @@ class BinaryDataController @Inject()( datasetName: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -301,7 +301,7 @@ class BinaryDataController @Inject()( datasetName: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala index 062adc19c8b..0ee4cbfcd67 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala @@ -28,7 +28,7 @@ class DSMeshController @Inject()( datasetName: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { meshFiles <- meshFileService.exploreMeshFiles(organizationId, datasetName, dataLayerName) } yield Ok(Json.toJson(meshFiles)) @@ -48,7 +48,7 @@ class DSMeshController @Inject()( targetMappingName: Option[String], editableMappingTracingId: Option[String]): Action[ListMeshChunksRequest] = Action.async(validateJson[ListMeshChunksRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { _ <- Fox.successful(()) mappingNameForMeshFile = meshFileService.mappingNameForMeshFile(organizationId, @@ -79,7 +79,7 @@ class DSMeshController @Inject()( datasetName: String, dataLayerName: String): Action[MeshChunkDataRequestList] = Action.async(validateJson[MeshChunkDataRequestList]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (data, encoding) <- meshFileService.readMeshChunk(organizationId, datasetName, dataLayerName, request.body) ?~> "mesh.file.loadChunk.failed" } yield { @@ -95,7 +95,7 @@ class DSMeshController @Inject()( datasetName: String, dataLayerName: String): Action[FullMeshRequest] = Action.async(validateJson[FullMeshRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { data: Array[Byte] <- fullMeshService.loadFor(organizationId, datasetName, dataLayerName, request.body) ?~> "mesh.file.loadChunk.failed" diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala index ff5befccdc3..24b2c4f511f 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala @@ -68,7 +68,7 @@ class DataSourceController @Inject()( def readInboxDataSource(token: Option[String], organizationId: String, datasetName: String): Action[AnyContent] = Action.async { implicit request => { - accessTokenService.validateAccessForSyncBlock( + accessTokenService.validateAccessFromTokenContextForSyncBlock( UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { // Read directly from file, not from repository to ensure recent changes are seen val dataSource: InboxDataSource = @@ -81,7 +81,7 @@ class DataSourceController @Inject()( } def triggerInboxCheckBlocking(token: Option[String]): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.administrateDataSources) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDataSources) { for { _ <- dataSourceService.checkInbox(verbose = true) } yield Ok @@ -90,7 +90,7 @@ class DataSourceController @Inject()( def reserveUpload(token: Option[String]): Action[ReserveUploadInformation] = Action.async(validateJson[ReserveUploadInformation]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(request.body.organization)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDataSources(request.body.organization)) { for { isKnownUpload <- uploadService.isKnownUpload(request.body.uploadId) _ <- if (!isKnownUpload) { @@ -103,7 +103,7 @@ class DataSourceController @Inject()( def getUnfinishedUploads(token: Option[String], organizationName: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(organizationName)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDataSources(organizationName)) { for { unfinishedUploads <- remoteWebknossosClient.getUnfinishedUploadsForUser(organizationName) unfinishedUploadsWithUploadIds <- uploadService.addUploadIdsToUnfinishedUploads(unfinishedUploads) @@ -115,7 +115,7 @@ class DataSourceController @Inject()( // and it can be put in a webknossos folder where they have access def reserveManualUpload(token: Option[String]): Action[ReserveManualUploadInformation] = Action.async(validateJson[ReserveManualUploadInformation]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(request.body.organization)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDataSources(request.body.organization)) { for { _ <- remoteWebknossosClient.reserveDataSourceUpload( ReserveUploadInformation( @@ -166,7 +166,7 @@ class DataSourceController @Inject()( for { dataSourceId <- uploadService.getDataSourceIdByUploadId( uploadService.extractDatasetUploadId(uploadFileId)) ?~> "dataset.upload.validation.failed" - result <- accessTokenService.validateAccess(UserAccessRequest.writeDataSource(dataSourceId)) { + result <- accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeDataSource(dataSourceId)) { for { isKnownUpload <- uploadService.isKnownUploadByFileId(uploadFileId) _ <- bool2Fox(isKnownUpload) ?~> "dataset.upload.validation.failed" @@ -188,7 +188,7 @@ class DataSourceController @Inject()( for { dataSourceId <- uploadService.getDataSourceIdByUploadId( uploadService.extractDatasetUploadId(resumableIdentifier)) ?~> "dataset.upload.validation.failed" - result <- accessTokenService.validateAccess(UserAccessRequest.writeDataSource(dataSourceId)) { + result <- accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeDataSource(dataSourceId)) { for { isKnownUpload <- uploadService.isKnownUploadByFileId(resumableIdentifier) _ <- bool2Fox(isKnownUpload) ?~> "dataset.upload.validation.failed" @@ -204,7 +204,7 @@ class DataSourceController @Inject()( for { dataSourceId <- uploadService .getDataSourceIdByUploadId(request.body.uploadId) ?~> "dataset.upload.validation.failed" - result <- accessTokenService.validateAccess(UserAccessRequest.writeDataSource(dataSourceId)) { + result <- accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeDataSource(dataSourceId)) { for { (dataSourceId, datasetSizeBytes) <- uploadService.finishUpload(request.body) ?~> "finishUpload.failed" _ <- remoteWebknossosClient.reportUpload(dataSourceId, @@ -224,7 +224,7 @@ class DataSourceController @Inject()( case true => uploadService.getDataSourceIdByUploadId(request.body.uploadId) } dataSourceIdFox.flatMap { dataSourceId => - accessTokenService.validateAccess(UserAccessRequest.deleteDataSource(dataSourceId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.deleteDataSource(dataSourceId)) { for { _ <- remoteWebknossosClient.deleteDataSource(dataSourceId) ?~> "dataset.delete.webknossos.failed" _ <- uploadService.cancelUpload(request.body) ?~> "Could not cancel the upload." @@ -239,7 +239,7 @@ class DataSourceController @Inject()( datasetName: String, dataLayerName: String ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessForSyncBlock( + accessTokenService.validateAccessFromTokenContextForSyncBlock( UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { addNoCacheHeaderFallback( Ok(Json.toJson(dataSourceService.exploreMappings(organizationId, datasetName, dataLayerName)))) @@ -252,7 +252,7 @@ class DataSourceController @Inject()( datasetName: String, dataLayerName: String ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox agglomerateList = agglomerateService.exploreAgglomerates(organizationId, datasetName, dataLayerName) @@ -268,7 +268,7 @@ class DataSourceController @Inject()( mappingName: String, agglomerateId: Long ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox skeleton <- agglomerateService.generateSkeleton(organizationId, @@ -288,7 +288,7 @@ class DataSourceController @Inject()( mappingName: String, agglomerateId: Long ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox agglomerateGraph <- agglomerateService.generateAgglomerateGraph( @@ -306,7 +306,7 @@ class DataSourceController @Inject()( mappingName: String, segmentId: Long ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox position <- agglomerateService.positionForSegmentId( @@ -323,7 +323,7 @@ class DataSourceController @Inject()( dataLayerName: String, mappingName: String ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox largestAgglomerateId: Long <- agglomerateService @@ -347,7 +347,7 @@ class DataSourceController @Inject()( dataLayerName: String, mappingName: String ): Action[ListOfLong] = Action.async(validateProto[ListOfLong]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox agglomerateIds: Seq[Long] <- agglomerateService @@ -372,7 +372,7 @@ class DataSourceController @Inject()( dataLayerName: String, mappingName: String ): Action[ListOfLong] = Action.async(validateProto[ListOfLong]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox agglomerateIds: Array[Long] <- agglomerateService @@ -391,7 +391,7 @@ class DataSourceController @Inject()( def update(token: Option[String], organizationId: String, datasetName: String): Action[DataSource] = Action.async(validateJson[DataSource]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.writeDataSource(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeDataSource(DataSourceId(datasetName, organizationId))) { for { _ <- Fox.successful(()) dataSource <- dataSourceRepository.find(DataSourceId(datasetName, organizationId)).toFox ?~> Messages( @@ -407,7 +407,7 @@ class DataSourceController @Inject()( datasetName: String, folderId: Option[String]): Action[DataSource] = Action.async(validateJson[DataSource]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.administrateDataSources) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDataSources) { for { _ <- bool2Fox(dataSourceRepository.find(DataSourceId(datasetName, organizationId)).isEmpty) ?~> Messages( "dataSource.alreadyPresent") @@ -435,7 +435,7 @@ class DataSourceController @Inject()( def createOrganizationDirectory(token: Option[String], organizationId: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessForSyncBlock(UserAccessRequest.administrateDataSources(organizationId)) { + accessTokenService.validateAccessFromTokenContextForSyncBlock(UserAccessRequest.administrateDataSources(organizationId)) { val newOrganizationDirectory = new File(f"${dataSourceService.dataBaseDir}/$organizationId") newOrganizationDirectory.mkdirs() if (newOrganizationDirectory.isDirectory) @@ -450,7 +450,7 @@ class DataSourceController @Inject()( datasetName: Option[String] = None): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(organizationId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDataSources(organizationId)) { for { before <- Fox.successful(System.currentTimeMillis()) usedStorageInBytes: List[DirectoryStorageReport] <- storageUsageService.measureStorage(organizationId, @@ -470,7 +470,7 @@ class DataSourceController @Inject()( datasetName: String, layerName: Option[String] = None): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(organizationId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDataSources(organizationId)) { val (closedAgglomerateFileHandleCount, clearedBucketProviderCount, removedChunksCount) = binaryDataServiceHolder.binaryDataService.clearCache(organizationId, datasetName, layerName) val reloadedDataSource = dataSourceService.dataSourceFromDir( @@ -493,7 +493,7 @@ class DataSourceController @Inject()( def deleteOnDisk(token: Option[String], organizationId: String, datasetName: String): Action[AnyContent] = Action.async { implicit request => val dataSourceId = DataSourceId(datasetName, organizationId) - accessTokenService.validateAccess(UserAccessRequest.deleteDataSource(dataSourceId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.deleteDataSource(dataSourceId)) { for { _ <- binaryDataServiceHolder.binaryDataService.deleteOnDisk( organizationId, @@ -506,7 +506,7 @@ class DataSourceController @Inject()( def compose(token: Option[String]): Action[ComposeRequest] = Action.async(validateJson[ComposeRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(request.body.organizationId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDataSources(request.body.organizationId)) { for { _ <- Fox.serialCombined(request.body.layers.map(_.datasetId).toList)(id => accessTokenService.assertUserAccess( @@ -522,7 +522,7 @@ class DataSourceController @Inject()( datasetName: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { val connectomeFileNames = connectomeFileService.exploreConnectomeFiles(organizationId, datasetName, dataLayerName) for { @@ -543,7 +543,7 @@ class DataSourceController @Inject()( datasetName: String, dataLayerName: String): Action[ByAgglomerateIdsRequest] = Action.async(validateJson[ByAgglomerateIdsRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { meshFilePath <- Fox.successful( connectomeFileService @@ -559,7 +559,7 @@ class DataSourceController @Inject()( dataLayerName: String, direction: String): Action[BySynapseIdsRequest] = Action.async(validateJson[BySynapseIdsRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { meshFilePath <- Fox.successful( connectomeFileService @@ -576,7 +576,7 @@ class DataSourceController @Inject()( datasetName: String, dataLayerName: String): Action[BySynapseIdsRequest] = Action.async(validateJson[BySynapseIdsRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { meshFilePath <- Fox.successful( connectomeFileService @@ -591,7 +591,7 @@ class DataSourceController @Inject()( datasetName: String, dataLayerName: String): Action[BySynapseIdsRequest] = Action.async(validateJson[BySynapseIdsRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { meshFilePath <- Fox.successful( connectomeFileService @@ -606,7 +606,7 @@ class DataSourceController @Inject()( dataSetName: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(dataSetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(dataSetName, organizationId))) { val segmentIndexFileOpt = segmentIndexFileService.getSegmentIndexFile(organizationId, dataSetName, dataLayerName).toOption Future.successful(Ok(Json.toJson(segmentIndexFileOpt.isDefined))) @@ -623,7 +623,7 @@ class DataSourceController @Inject()( dataLayerName: String, segmentId: String): Action[GetSegmentIndexParameters] = Action.async(validateJson[GetSegmentIndexParameters]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { segmentIds <- segmentIdsForAgglomerateIdIfNeeded( organizationId, @@ -660,7 +660,7 @@ class DataSourceController @Inject()( datasetName: String, dataLayerName: String): Action[GetMultipleSegmentIndexParameters] = Action.async(validateJson[GetMultipleSegmentIndexParameters]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { segmentIdsAndBucketPositions <- Fox.serialCombined(request.body.segmentIds) { segmentOrAgglomerateId => for { @@ -692,7 +692,7 @@ class DataSourceController @Inject()( datasetName: String, dataLayerName: String): Action[SegmentStatisticsParameters] = Action.async(validateJson[SegmentStatisticsParameters]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { _ <- segmentIndexFileService.assertSegmentIndexFileExists(organizationId, datasetName, dataLayerName) volumes <- Fox.serialCombined(request.body.segmentIds) { segmentId => @@ -714,7 +714,7 @@ class DataSourceController @Inject()( datasetName: String, dataLayerName: String): Action[SegmentStatisticsParameters] = Action.async(validateJson[SegmentStatisticsParameters]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { _ <- segmentIndexFileService.assertSegmentIndexFileExists(organizationId, datasetName, dataLayerName) boxes <- Fox.serialCombined(request.body.segmentIds) { segmentId => @@ -732,7 +732,7 @@ class DataSourceController @Inject()( // Called directly by wk side def exploreRemoteDataset(token: Option[String]): Action[ExploreRemoteDatasetRequest] = Action.async(validateJson[ExploreRemoteDatasetRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(request.body.organizationId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDataSources(request.body.organizationId)) { val reportMutable = ListBuffer[String]() val hasLocalFilesystemRequest = request.body.layerParameters.exists(param => new URI(param.remoteUri).getScheme == DataVaultService.schemeFile) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ExportsController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ExportsController.scala index 009ac58d0f5..efdac2ed607 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ExportsController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ExportsController.scala @@ -36,7 +36,7 @@ class ExportsController @Inject()(webknossosClient: DSRemoteWebknossosClient, override def allowRemoteOrigin: Boolean = true def download(token: Option[String], jobId: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.downloadJobExport(jobId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.downloadJobExport(jobId)) { for { exportProperties <- webknossosClient.getJobExportProperties(jobId) fullPath = exportProperties.fullPathIn(dataBaseDir) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala index 1f86aef3d08..a04a7d81c40 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala @@ -56,7 +56,7 @@ class ZarrStreamingController @Inject()( datasetName: String, dataLayerName: String = "", ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -73,7 +73,7 @@ class ZarrStreamingController @Inject()( datasetName: String, dataLayerName: String = "", ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -151,7 +151,7 @@ class ZarrStreamingController @Inject()( datasetName: String, zarrVersion: Int, ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { dataSource <- dataSourceRepository.findUsable(DataSourceId(datasetName, organizationId)).toFox ~> NOT_FOUND dataLayers = dataSource.dataLayers @@ -232,7 +232,7 @@ class ZarrStreamingController @Inject()( mag: String, coordinates: String, ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { rawZarrCube(organizationId, datasetName, dataLayerName, mag, coordinates) } } @@ -298,7 +298,7 @@ class ZarrStreamingController @Inject()( dataLayerName: String, mag: String, ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { zArray(organizationId, datasetName, dataLayerName, mag) } } @@ -319,7 +319,7 @@ class ZarrStreamingController @Inject()( dataLayerName: String, mag: String, ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { zarrJsonForMag(organizationId, datasetName, dataLayerName, mag) } } @@ -389,7 +389,7 @@ class ZarrStreamingController @Inject()( mag: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { dataLayerMagFolderContents(organizationId, datasetName, dataLayerName, mag, zarrVersion) } } @@ -450,7 +450,7 @@ class ZarrStreamingController @Inject()( datasetName: String, dataLayerName: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { dataLayerFolderContents(organizationId, datasetName, dataLayerName, zarrVersion) } } @@ -507,7 +507,7 @@ class ZarrStreamingController @Inject()( datasetName: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { dataSource <- dataSourceRepository.findUsable(DataSourceId(datasetName, organizationId)).toFox ?~> Messages( "dataSource.notFound") ~> NOT_FOUND @@ -554,7 +554,7 @@ class ZarrStreamingController @Inject()( organizationId: String, datasetName: String, dataLayerName: String = ""): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessForSyncBlock( + accessTokenService.validateAccessFromTokenContextForSyncBlock( UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { Ok(zGroupJson) } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AccessTokenService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AccessTokenService.scala index 52b56268077..b44d93c0468 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AccessTokenService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AccessTokenService.scala @@ -67,14 +67,14 @@ trait AccessTokenService { private lazy val accessAnswersCache: AlfuCache[(UserAccessRequest, Option[String]), UserAccessAnswer] = AlfuCache(timeToLive = AccessExpiration, timeToIdle = AccessExpiration) - def validateAccessForSyncBlock(accessRequest: UserAccessRequest)(block: => Result)(implicit ec: ExecutionContext, - tc: TokenContext): Fox[Result] = - validateAccess(accessRequest) { + def validateAccessFromTokenContextForSyncBlock(accessRequest: UserAccessRequest)( + block: => Result)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Result] = + validateAccessFromTokenContext(accessRequest) { Future.successful(block) } - def validateAccess(accessRequest: UserAccessRequest)(block: => Future[Result])(implicit ec: ExecutionContext, - tc: TokenContext): Fox[Result] = + def validateAccessFromTokenContext(accessRequest: UserAccessRequest)( + block: => Future[Result])(implicit ec: ExecutionContext, tc: TokenContext): Fox[Result] = for { userAccessAnswer <- hasUserAccess(accessRequest) ?~> "Failed to check data access, token may be expired, consider reloading." result <- executeBlockOnPositiveAnswer(userAccessAnswer, block) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index aad111d9e9b..ced731c2e80 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -50,6 +50,11 @@ case class AnnotationWithTracings( (info, _) <- editableMappingsByTracingId.get(tracingId) } yield info + def getEditableMappingUpdater(tracingId: String): Option[EditableMappingUpdater] = + for { + (_, updater) <- editableMappingsByTracingId.get(tracingId) + } yield updater + def version: Long = annotation.version def addTracing(a: AddLayerAnnotationUpdateAction): AnnotationWithTracings = @@ -113,5 +118,7 @@ case class AnnotationWithTracings( def applyEditableMappingAction(a: EditableMappingUpdateAction)( implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = - Fox.failure("not implemented yet") // TODO + for { + updater <- getEditableMappingUpdater("tracingId") // TODO editable mapping update actions need tracing id + } yield this // TODO } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 73decf3036d..eb422d9fff1 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -8,6 +8,7 @@ import com.scalableminds.webknossos.datastore.Annotation.AnnotationProto import com.scalableminds.webknossos.datastore.EditableMappingInfo.EditableMappingInfo import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing +import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing.ElementClassProto import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ EditableMappingService, EditableMappingUpdateAction, @@ -24,8 +25,16 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ BucketMutatingVolumeUpdateAction, VolumeUpdateAction } -import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} -import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingUpdatesReport} +import com.scalableminds.webknossos.tracingstore.tracings.{ + KeyValueStoreImplicits, + RemoteFallbackLayer, + TracingDataStore +} +import com.scalableminds.webknossos.tracingstore.{ + TSRemoteDatastoreClient, + TSRemoteWebknossosClient, + TracingUpdatesReport +} import com.typesafe.scalalogging.LazyLogging import net.liftweb.common.{Empty, Full} import play.api.libs.json.{JsObject, JsValue, Json} @@ -34,6 +43,8 @@ import javax.inject.Inject import scala.concurrent.ExecutionContext class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosClient, + editableMappingService: EditableMappingService, + remoteDatastoreClient: TSRemoteDatastoreClient, tracingDataStore: TracingDataStore) extends KeyValueStoreImplicits with LazyLogging { @@ -159,13 +170,18 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl updates, requestedSkeletonTracingIds, requestedVolumeTracingIds) ?~> "findTracingsForUpdates.failed" - annotationWithTracingsAndMappings <- findEditableMappingsForUpdates(annotationWithTracings, updates) + annotationWithTracingsAndMappings <- findEditableMappingsForUpdates(annotationWithTracings, + updates, + annotation.version, + targetVersion) updated <- applyUpdates(annotationWithTracingsAndMappings, annotationId, updates, targetVersion) ?~> "applyUpdates.inner.failed" } yield updated private def findEditableMappingsForUpdates( // TODO integrate with findTracings? annotationWithTracings: AnnotationWithTracings, - updates: List[UpdateAction])(implicit ec: ExecutionContext) = { + updates: List[UpdateAction], + currentMaterializedVersion: Long, + targetVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext) = { val volumeIdsWithEditableMapping = annotationWithTracings.volumesIdsThatHaveEditableMapping // TODO intersect with editable mapping updates? for { @@ -176,11 +192,36 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl } yield annotationWithTracings.copy( editableMappingsByTracingId = editableMappingInfos - .map(keyValuePair => (keyValuePair.key, (keyValuePair.value, editableMappingUpdaterFor(keyValuePair.value)))) + .map( + keyValuePair => + (keyValuePair.key, + (keyValuePair.value, + editableMappingUpdaterFor(keyValuePair.key, + keyValuePair.value, + currentMaterializedVersion, + targetVersion)))) .toMap) } - def editableMappingUpdaterFor(editableMappingInfo: EditableMappingInfo): EditableMappingUpdater = ??? // TODO + private def editableMappingUpdaterFor(tracingId: String, + editableMappingInfo: EditableMappingInfo, + currentMaterializedVersion: Long, + targetVersion: Long)(implicit tc: TokenContext): EditableMappingUpdater = { + val remoteFallbackLayer + : RemoteFallbackLayer = RemoteFallbackLayer("todo", "todo", "todo", ElementClassProto.uint8) // TODO + new EditableMappingUpdater( + tracingId, + editableMappingInfo.baseMappingName, + currentMaterializedVersion, + targetVersion, + remoteFallbackLayer, + tc, + remoteDatastoreClient, + editableMappingService, + tracingDataStore, + relyOnAgglomerateIds = false // TODO + ) + } private def findTracingsForUpdates( annotation: AnnotationProto, @@ -217,11 +258,10 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl } yield AnnotationWithTracings(annotation, skeletonTracingsMap ++ volumeTracingsMap, editableMappingsMap) } - private def applyUpdates( - annotation: AnnotationWithTracings, - annotationId: String, - updates: List[UpdateAction], - targetVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = { + private def applyUpdates(annotation: AnnotationWithTracings, + annotationId: String, + updates: List[UpdateAction], + targetVersion: Long)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = { def updateIter(annotationWithTracingsFox: Fox[AnnotationWithTracings], remainingUpdates: List[UpdateAction]): Fox[AnnotationWithTracings] = diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala index a8a9484ba11..711748b2dda 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala @@ -40,7 +40,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer def makeMappingEditable(token: Option[String], annotationId: String, tracingId: String): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- volumeTracingService.find(annotationId, tracingId) tracingMappingName <- tracing.mappingName ?~> "annotation.noMappingSet" @@ -112,7 +112,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer version: Option[Long]): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- volumeTracingService.find(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) @@ -128,7 +128,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer tracingId: String, agglomerateId: Long): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- volumeTracingService.find(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) @@ -150,7 +150,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer def agglomerateIdsForSegments(token: Option[String], annotationId: String, tracingId: String): Action[ListOfLong] = Action.async(validateProto[ListOfLong]) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- volumeTracingService.find(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) @@ -174,7 +174,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer def agglomerateGraphMinCut(token: Option[String], annotationId: String, tracingId: String): Action[MinCutParameters] = Action.async(validateJson[MinCutParameters]) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- volumeTracingService.find(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) @@ -190,7 +190,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer tracingId: String): Action[NeighborsParameters] = Action.async(validateJson[NeighborsParameters]) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- volumeTracingService.find(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala index 9b9f9f191a9..86feebacce9 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala @@ -38,7 +38,7 @@ class SkeletonTracingController @Inject()(val tracingService: SkeletonTracingSer def mergedFromContents(token: Option[String], persist: Boolean): Action[SkeletonTracings] = Action.async(validateProto[SkeletonTracings]) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.webknossos) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { val tracings: List[Option[SkeletonTracing]] = request.body for { mergedTracing <- Fox.box2Fox(tracingService.merge(tracings.flatten, MergedVolumeStats.empty(), Empty)) @@ -59,7 +59,7 @@ class SkeletonTracingController @Inject()(val tracingService: SkeletonTracingSer boundingBox: Option[String]): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.webknossos) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { for { tracing <- tracingService.find(annotationId, tracingId, version, applyUpdates = true) ?~> Messages( "tracing.notFound") diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index d398abbdc4f..e1ab8ec7070 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -29,7 +29,7 @@ class TSAnnotationController @Inject()( def save(token: Option[String], annotationId: String): Action[AnnotationProto] = Action.async(validateProto[AnnotationProto]) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.webknossos) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { for { // TODO assert id does not already exist _ <- tracingDataStore.annotations.put(annotationId, 0L, request.body) @@ -43,7 +43,7 @@ class TSAnnotationController @Inject()( Action.async(validateJson[List[UpdateActionGroup]]) { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { - accessTokenService.validateAccess(UserAccessRequest.writeAnnotation(annotationId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeAnnotation(annotationId)) { for { _ <- annotationTransactionService.handleUpdateGroups(annotationId, request.body) } yield Ok @@ -57,7 +57,7 @@ class TSAnnotationController @Inject()( newestVersion: Option[Long] = None, oldestVersion: Option[Long] = None): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readAnnotation(annotationId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readAnnotation(annotationId)) { for { updateLog <- annotationService.updateActionLog(annotationId, newestVersion, oldestVersion) } yield Ok(updateLog) @@ -68,7 +68,7 @@ class TSAnnotationController @Inject()( def newestVersion(token: Option[String], annotationId: String): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readAnnotation(annotationId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readAnnotation(annotationId)) { for { newestVersion <- annotationService.currentMaterializableVersion(annotationId) } yield JsonOk(Json.obj("version" -> newestVersion)) @@ -79,7 +79,7 @@ class TSAnnotationController @Inject()( def updateActionStatistics(token: Option[String], tracingId: String): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { statistics <- annotationService.updateActionStatistics(tracingId) } yield Ok(statistics) @@ -91,7 +91,7 @@ class TSAnnotationController @Inject()( Action.async { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { - accessTokenService.validateAccess(UserAccessRequest.readAnnotation(annotationId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readAnnotation(annotationId)) { for { annotationProto <- annotationService.get(annotationId, version) } yield Ok(annotationProto.toByteArray).as(protobufMimeType) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala index 4125161aaf1..8bc7d999ea1 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala @@ -44,7 +44,7 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C def save(token: Option[String]): Action[T] = Action.async(validateProto[T]) { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { - accessTokenService.validateAccess(UserAccessRequest.webknossos) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { val tracing = request.body tracingService.save(tracing, None, 0).map { newId => Ok(Json.toJson(newId)) @@ -57,7 +57,7 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C def saveMultiple(token: Option[String]): Action[Ts] = Action.async(validateProto[Ts]) { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { - accessTokenService.validateAccess(UserAccessRequest.webknossos) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { val savedIds = Fox.sequence(request.body.map { tracingOpt: Option[T] => tracingOpt match { case Some(tracing) => tracingService.save(tracing, None, 0).map(Some(_)) @@ -73,7 +73,7 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C def get(token: Option[String], annotationId: String, tracingId: String, version: Option[Long]): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId, version, applyUpdates = true) ?~> Messages( "tracing.notFound") @@ -85,7 +85,7 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C def getMultiple(token: Option[String]): Action[List[Option[TracingSelector]]] = Action.async(validateJson[List[Option[TracingSelector]]]) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.webknossos) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { for { tracings <- tracingService.findMultiple(request.body, applyUpdates = true) } yield { @@ -98,7 +98,7 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C def mergedFromIds(token: Option[String], persist: Boolean): Action[List[Option[TracingSelector]]] = Action.async(validateJson[List[Option[TracingSelector]]]) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.webknossos) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { for { tracingOpts <- tracingService.findMultiple(request.body, applyUpdates = true) ?~> Messages( "tracing.notFound") diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 7d3d5871167..ddba192d7bb 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -86,7 +86,7 @@ class VolumeTracingController @Inject()( Action.async { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { - accessTokenService.validateAccess(UserAccessRequest.webknossos) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { for { initialData <- request.body.asRaw.map(_.asFile) ?~> Messages("zipFile.notFound") tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") @@ -104,7 +104,7 @@ class VolumeTracingController @Inject()( def mergedFromContents(token: Option[String], persist: Boolean): Action[VolumeTracings] = Action.async(validateProto[VolumeTracings]) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.webknossos) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { for { _ <- Fox.successful(()) tracings = request.body @@ -124,7 +124,7 @@ class VolumeTracingController @Inject()( Action.async { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { - accessTokenService.validateAccess(UserAccessRequest.webknossos) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { for { initialData <- request.body.asRaw.map(_.asFile) ?~> Messages("zipFile.notFound") tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") @@ -147,7 +147,7 @@ class VolumeTracingController @Inject()( voxelSizeUnit: Option[String]): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId, version) ?~> Messages("tracing.notFound") volumeDataZipFormatParsed <- VolumeDataZipFormat.fromString(volumeDataZipFormat).toFox @@ -169,7 +169,7 @@ class VolumeTracingController @Inject()( def data(token: Option[String], annotationId: String, tracingId: String): Action[List[WebknossosDataRequest]] = Action.async(validateJson[List[WebknossosDataRequest]]) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") (data, indices) <- if (tracing.getHasEditableMapping) @@ -198,7 +198,7 @@ class VolumeTracingController @Inject()( boundingBox: Option[String]): Action[AnyContent] = Action.async { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { - accessTokenService.validateAccess(UserAccessRequest.webknossos) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { for { tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") _ = logger.info(s"Duplicating volume tracing $tracingId...") @@ -237,7 +237,7 @@ class VolumeTracingController @Inject()( tracingId: String): Action[MultipartFormData[TemporaryFile]] = Action.async(parse.multipartFormData) { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.writeTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") currentVersion <- request.body.dataParts("currentVersion").headOption.flatMap(_.toIntOpt).toFox @@ -258,7 +258,7 @@ class VolumeTracingController @Inject()( dryRun: Boolean): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.webknossos) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { for { tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") currentVersion <- annotationService.currentMaterializableVersion(tracingId) @@ -284,7 +284,7 @@ class VolumeTracingController @Inject()( newestVersion: Option[Long] = None, oldestVersion: Option[Long] = None): Action[AnyContent] = Action.async { implicit request => log() { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { updateLog <- tracingService.updateActionLog(tracingId, newestVersion, oldestVersion) } yield Ok(updateLog) @@ -296,7 +296,7 @@ class VolumeTracingController @Inject()( annotationId: String, tracingId: String): Action[WebknossosAdHocMeshRequest] = Action.async(validateJson[WebknossosAdHocMeshRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { // The client expects the ad-hoc mesh as a flat float-array. Three consecutive floats form a 3D point, three // consecutive 3D points (i.e., nine floats) form a triangle. @@ -316,7 +316,7 @@ class VolumeTracingController @Inject()( def loadFullMeshStl(token: Option[String], annotationId: String, tracingId: String): Action[FullMeshRequest] = Action.async(validateJson[FullMeshRequest]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { data: Array[Byte] <- fullMeshService.loadFor(annotationId, tracingId, request.body) ?~> "mesh.file.loadChunk.failed" } yield Ok(data) @@ -331,7 +331,7 @@ class VolumeTracingController @Inject()( def findData(token: Option[String], annotationId: String, tracingId: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { positionOpt <- tracingService.findData(annotationId, tracingId) } yield { @@ -345,7 +345,7 @@ class VolumeTracingController @Inject()( tracingId: String, agglomerateId: Long): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) _ <- bool2Fox(tracing.getHasEditableMapping) ?~> "Cannot query agglomerate skeleton for volume annotation" @@ -362,7 +362,7 @@ class VolumeTracingController @Inject()( annotationId: String, tracingId: String): Action[SegmentStatisticsParameters] = Action.async(validateJson[SegmentStatisticsParameters]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) mappingName <- tracingService.baseMappingName(tracing) @@ -382,7 +382,7 @@ class VolumeTracingController @Inject()( annotationId: String, tracingId: String): Action[SegmentStatisticsParameters] = Action.async(validateJson[SegmentStatisticsParameters]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) mappingName <- tracingService.baseMappingName(tracing) @@ -403,7 +403,7 @@ class VolumeTracingController @Inject()( tracingId: String, segmentId: Long): Action[GetSegmentIndexParameters] = Action.async(validateJson[GetSegmentIndexParameters]) { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { fallbackLayer <- tracingService.getFallbackLayer(annotationId, tracingId) tracing <- tracingService.find(annotationId, tracingId) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala index 856e0545ecb..488c0c14c54 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala @@ -62,7 +62,7 @@ class VolumeTracingZarrStreamingController @Inject()( tracingId: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) @@ -84,7 +84,7 @@ class VolumeTracingZarrStreamingController @Inject()( tracingId: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto(_).toMagLiteral(allowScalar = true)) @@ -101,7 +101,7 @@ class VolumeTracingZarrStreamingController @Inject()( mag: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND @@ -125,7 +125,7 @@ class VolumeTracingZarrStreamingController @Inject()( mag: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) @@ -138,7 +138,7 @@ class VolumeTracingZarrStreamingController @Inject()( def zArray(token: Option[String], annotationId: String, tracingId: String, mag: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) @@ -172,7 +172,7 @@ class VolumeTracingZarrStreamingController @Inject()( def zarrJsonForMag(token: Option[String], annotationId: String, tracingId: String, mag: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND @@ -218,7 +218,7 @@ class VolumeTracingZarrStreamingController @Inject()( def zGroup(token: Option[String], annotationId: String, tracingId: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { Future(Ok(Json.toJson(NgffGroupHeader(zarr_format = 2)))) } } @@ -233,7 +233,7 @@ class VolumeTracingZarrStreamingController @Inject()( annotationId: String, tracingId: String, ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND @@ -251,7 +251,7 @@ class VolumeTracingZarrStreamingController @Inject()( annotationId: String, tracingId: String, ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND @@ -272,7 +272,7 @@ class VolumeTracingZarrStreamingController @Inject()( tracingName: Option[String], zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND @@ -297,7 +297,7 @@ class VolumeTracingZarrStreamingController @Inject()( coordinates: String): Action[AnyContent] = Action.async { implicit request => { - accessTokenService.validateAccess(UserAccessRequest.readTracing(tracingId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 7d2f76dc0ed..c48e6ec3abe 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -271,6 +271,7 @@ class VolumeTracingService @Inject()( bool2Fox(mag.isIsotropic) } + /* // TODO private def revertToVolumeVersion(annotationId: String, tracingId: String, @@ -332,6 +333,7 @@ class VolumeTracingService @Inject()( _ <- segmentIndexBuffer.flush() } yield sourceTracing } + */ def initializeWithDataMultiple(annotationId: String, tracingId: String, tracing: VolumeTracing, initialData: File)( implicit mp: MessagesProvider, From 8883306f7d0a86226b23163f64e39696ea6c7219 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 18 Sep 2024 11:46:23 +0200 Subject: [PATCH 066/150] remove token from parameter lists --- .../controllers/BinaryDataController.scala | 50 ++--- .../controllers/DSMeshController.scala | 36 ++- .../controllers/DataSourceController.scala | 206 +++++++++--------- .../controllers/ExportsController.scala | 2 +- .../controllers/ZarrStreamingController.scala | 153 ++++++------- ....scalableminds.webknossos.datastore.routes | 190 ++++++++-------- .../EditableMappingController.scala | 57 +++-- .../SkeletonTracingController.scala | 5 +- .../controllers/TSAnnotationController.scala | 39 ++-- .../controllers/TracingController.scala | 10 +- .../controllers/VolumeTracingController.scala | 66 ++---- ...VolumeTracingZarrStreamingController.scala | 40 +--- ...alableminds.webknossos.tracingstore.routes | 118 +++++----- 13 files changed, 445 insertions(+), 527 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala index 514ea193faa..02a48218fef 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala @@ -48,12 +48,12 @@ class BinaryDataController @Inject()( val adHocMeshService: AdHocMeshService = adHocMeshServiceHolder.dataStoreAdHocMeshService def requestViaWebknossos( - token: Option[String], organizationId: String, datasetName: String, dataLayerName: String ): Action[List[WebknossosDataRequest]] = Action.async(validateJson[List[WebknossosDataRequest]]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { logTime(slackNotificationService.noticeSlowRequest) { val t = Instant.now for { @@ -77,7 +77,6 @@ class BinaryDataController @Inject()( * Handles requests for raw binary data via HTTP GET. */ def requestRawCuboid( - token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, @@ -95,7 +94,8 @@ class BinaryDataController @Inject()( halfByte: Boolean, mappingName: Option[String] ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -114,12 +114,12 @@ class BinaryDataController @Inject()( } def requestRawCuboidPost( - token: Option[String], organizationId: String, datasetName: String, dataLayerName: String ): Action[RawCuboidRequest] = Action.async(validateJson[RawCuboidRequest]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -132,8 +132,7 @@ class BinaryDataController @Inject()( /** * Handles a request for raw binary data via a HTTP GET. Used by knossos. */ - def requestViaKnossos(token: Option[String], - organizationId: String, + def requestViaKnossos(organizationId: String, datasetName: String, dataLayerName: String, resolution: Int, @@ -141,7 +140,8 @@ class BinaryDataController @Inject()( y: Int, z: Int, cubeSize: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -160,8 +160,7 @@ class BinaryDataController @Inject()( } } - def thumbnailJpeg(token: Option[String], - organizationId: String, + def thumbnailJpeg(organizationId: String, datasetName: String, dataLayerName: String, x: Int, @@ -175,7 +174,8 @@ class BinaryDataController @Inject()( intensityMax: Option[Double], color: Option[String], invertColor: Option[Boolean]): Action[RawBuffer] = Action.async(parse.raw) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -216,13 +216,13 @@ class BinaryDataController @Inject()( } def mappingJson( - token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mappingName: String ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -237,12 +237,12 @@ class BinaryDataController @Inject()( /** * Handles ad-hoc mesh requests. */ - def requestAdHocMesh(token: Option[String], - organizationId: String, + def requestAdHocMesh(organizationId: String, datasetName: String, dataLayerName: String): Action[WebknossosAdHocMeshRequest] = Action.async(validateJson[WebknossosAdHocMeshRequest]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -278,12 +278,10 @@ class BinaryDataController @Inject()( private def formatNeighborList(neighbors: List[Int]): String = "[" + neighbors.mkString(", ") + "]" - def findData(token: Option[String], - organizationId: String, - datasetName: String, - dataLayerName: String): Action[AnyContent] = + def findData(organizationId: String, datasetName: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -296,12 +294,10 @@ class BinaryDataController @Inject()( } } - def histogram(token: Option[String], - organizationId: String, - datasetName: String, - dataLayerName: String): Action[AnyContent] = + def histogram(organizationId: String, datasetName: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala index 0ee4cbfcd67..eed1704178e 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DSMeshController.scala @@ -23,32 +23,30 @@ class DSMeshController @Inject()( override def allowRemoteOrigin: Boolean = true - def listMeshFiles(token: Option[String], - organizationId: String, - datasetName: String, - dataLayerName: String): Action[AnyContent] = + def listMeshFiles(organizationId: String, datasetName: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { meshFiles <- meshFileService.exploreMeshFiles(organizationId, datasetName, dataLayerName) } yield Ok(Json.toJson(meshFiles)) } } - def listMeshChunksForSegment(token: Option[String], - organizationId: String, + def listMeshChunksForSegment(organizationId: String, datasetName: String, dataLayerName: String, /* If targetMappingName is set, assume that meshfile contains meshes for - the oversegmentation. Collect mesh chunks of all *unmapped* segment ids - belonging to the supplied agglomerate id. - If it is not set, use meshfile as is, assume passed id is present in meshfile - Note: in case of an editable mapping, targetMappingName is its baseMapping name. + the oversegmentation. Collect mesh chunks of all *unmapped* segment ids + belonging to the supplied agglomerate id. + If it is not set, use meshfile as is, assume passed id is present in meshfile + Note: in case of an editable mapping, targetMappingName is its baseMapping name. */ targetMappingName: Option[String], editableMappingTracingId: Option[String]): Action[ListMeshChunksRequest] = Action.async(validateJson[ListMeshChunksRequest]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { _ <- Fox.successful(()) mappingNameForMeshFile = meshFileService.mappingNameForMeshFile(organizationId, @@ -74,12 +72,12 @@ class DSMeshController @Inject()( } } - def readMeshChunk(token: Option[String], - organizationId: String, + def readMeshChunk(organizationId: String, datasetName: String, dataLayerName: String): Action[MeshChunkDataRequestList] = Action.async(validateJson[MeshChunkDataRequestList]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (data, encoding) <- meshFileService.readMeshChunk(organizationId, datasetName, dataLayerName, request.body) ?~> "mesh.file.loadChunk.failed" } yield { @@ -90,12 +88,10 @@ class DSMeshController @Inject()( } } - def loadFullMeshStl(token: Option[String], - organizationId: String, - datasetName: String, - dataLayerName: String): Action[FullMeshRequest] = + def loadFullMeshStl(organizationId: String, datasetName: String, dataLayerName: String): Action[FullMeshRequest] = Action.async(validateJson[FullMeshRequest]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { data: Array[Byte] <- fullMeshService.loadFor(organizationId, datasetName, dataLayerName, request.body) ?~> "mesh.file.loadChunk.failed" diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala index 24b2c4f511f..7d130381413 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala @@ -65,7 +65,7 @@ class DataSourceController @Inject()( override def allowRemoteOrigin: Boolean = true - def readInboxDataSource(token: Option[String], organizationId: String, datasetName: String): Action[AnyContent] = + def readInboxDataSource(organizationId: String, datasetName: String): Action[AnyContent] = Action.async { implicit request => { accessTokenService.validateAccessFromTokenContextForSyncBlock( @@ -80,7 +80,7 @@ class DataSourceController @Inject()( } } - def triggerInboxCheckBlocking(token: Option[String]): Action[AnyContent] = Action.async { implicit request => + def triggerInboxCheckBlocking(): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDataSources) { for { _ <- dataSourceService.checkInbox(verbose = true) @@ -88,9 +88,10 @@ class DataSourceController @Inject()( } } - def reserveUpload(token: Option[String]): Action[ReserveUploadInformation] = + def reserveUpload(): Action[ReserveUploadInformation] = Action.async(validateJson[ReserveUploadInformation]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDataSources(request.body.organization)) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.administrateDataSources(request.body.organization)) { for { isKnownUpload <- uploadService.isKnownUpload(request.body.uploadId) _ <- if (!isKnownUpload) { @@ -101,7 +102,7 @@ class DataSourceController @Inject()( } } - def getUnfinishedUploads(token: Option[String], organizationName: String): Action[AnyContent] = + def getUnfinishedUploads(organizationName: String): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDataSources(organizationName)) { for { @@ -113,9 +114,10 @@ class DataSourceController @Inject()( // To be called by people with disk access but not DatasetManager role. This way, they can upload a dataset manually on disk, // and it can be put in a webknossos folder where they have access - def reserveManualUpload(token: Option[String]): Action[ReserveManualUploadInformation] = + def reserveManualUpload(): Action[ReserveManualUploadInformation] = Action.async(validateJson[ReserveManualUploadInformation]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDataSources(request.body.organization)) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.administrateDataSources(request.body.organization)) { for { _ <- remoteWebknossosClient.reserveDataSourceUpload( ReserveUploadInformation( @@ -147,7 +149,7 @@ class DataSourceController @Inject()( - As GET parameter: - token (string): datastore token identifying the uploading user */ - def uploadChunk(token: Option[String]): Action[MultipartFormData[Files.TemporaryFile]] = + def uploadChunk(): Action[MultipartFormData[Files.TemporaryFile]] = Action.async(parse.multipartFormData) { implicit request => val uploadForm = Form( tuple( @@ -166,7 +168,8 @@ class DataSourceController @Inject()( for { dataSourceId <- uploadService.getDataSourceIdByUploadId( uploadService.extractDatasetUploadId(uploadFileId)) ?~> "dataset.upload.validation.failed" - result <- accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeDataSource(dataSourceId)) { + result <- accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.writeDataSource(dataSourceId)) { for { isKnownUpload <- uploadService.isKnownUploadByFileId(uploadFileId) _ <- bool2Fox(isKnownUpload) ?~> "dataset.upload.validation.failed" @@ -183,7 +186,7 @@ class DataSourceController @Inject()( ) } - def testChunk(token: Option[String], resumableChunkNumber: Int, resumableIdentifier: String): Action[AnyContent] = + def testChunk(resumableChunkNumber: Int, resumableIdentifier: String): Action[AnyContent] = Action.async { implicit request => for { dataSourceId <- uploadService.getDataSourceIdByUploadId( @@ -198,26 +201,25 @@ class DataSourceController @Inject()( } yield result } - def finishUpload(token: Option[String]): Action[UploadInformation] = Action.async(validateJson[UploadInformation]) { - implicit request => - log() { - for { - dataSourceId <- uploadService - .getDataSourceIdByUploadId(request.body.uploadId) ?~> "dataset.upload.validation.failed" - result <- accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeDataSource(dataSourceId)) { - for { - (dataSourceId, datasetSizeBytes) <- uploadService.finishUpload(request.body) ?~> "finishUpload.failed" - _ <- remoteWebknossosClient.reportUpload(dataSourceId, - datasetSizeBytes, - request.body.needsConversion.getOrElse(false), - viaAddRoute = false) ?~> "reportUpload.failed" - } yield Ok - } - } yield result - } + def finishUpload(): Action[UploadInformation] = Action.async(validateJson[UploadInformation]) { implicit request => + log() { + for { + dataSourceId <- uploadService + .getDataSourceIdByUploadId(request.body.uploadId) ?~> "dataset.upload.validation.failed" + result <- accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeDataSource(dataSourceId)) { + for { + (dataSourceId, datasetSizeBytes) <- uploadService.finishUpload(request.body) ?~> "finishUpload.failed" + _ <- remoteWebknossosClient.reportUpload(dataSourceId, + datasetSizeBytes, + request.body.needsConversion.getOrElse(false), + viaAddRoute = false) ?~> "reportUpload.failed" + } yield Ok + } + } yield result + } } - def cancelUpload(token: Option[String]): Action[CancelUploadInformation] = + def cancelUpload(): Action[CancelUploadInformation] = Action.async(validateJson[CancelUploadInformation]) { implicit request => val dataSourceIdFox = uploadService.isKnownUpload(request.body.uploadId).flatMap { case false => Fox.failure("dataset.upload.validation.failed") @@ -234,7 +236,6 @@ class DataSourceController @Inject()( } def listMappings( - token: Option[String], organizationId: String, datasetName: String, dataLayerName: String @@ -247,12 +248,12 @@ class DataSourceController @Inject()( } def listAgglomerates( - token: Option[String], organizationId: String, datasetName: String, dataLayerName: String ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox agglomerateList = agglomerateService.exploreAgglomerates(organizationId, datasetName, dataLayerName) @@ -261,14 +262,14 @@ class DataSourceController @Inject()( } def generateAgglomerateSkeleton( - token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mappingName: String, agglomerateId: Long ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox skeleton <- agglomerateService.generateSkeleton(organizationId, @@ -281,14 +282,14 @@ class DataSourceController @Inject()( } def agglomerateGraph( - token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mappingName: String, agglomerateId: Long ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox agglomerateGraph <- agglomerateService.generateAgglomerateGraph( @@ -299,14 +300,14 @@ class DataSourceController @Inject()( } def positionForSegmentViaAgglomerateFile( - token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mappingName: String, segmentId: Long ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox position <- agglomerateService.positionForSegmentId( @@ -317,13 +318,13 @@ class DataSourceController @Inject()( } def largestAgglomerateId( - token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mappingName: String ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox largestAgglomerateId: Long <- agglomerateService @@ -341,13 +342,13 @@ class DataSourceController @Inject()( } def agglomerateIdsForSegmentIds( - token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mappingName: String ): Action[ListOfLong] = Action.async(validateProto[ListOfLong]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox agglomerateIds: Seq[Long] <- agglomerateService @@ -366,13 +367,13 @@ class DataSourceController @Inject()( } def agglomerateIdsForAllSegmentIds( - token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mappingName: String ): Action[ListOfLong] = Action.async(validateProto[ListOfLong]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { agglomerateService <- binaryDataServiceHolder.binaryDataService.agglomerateServiceOpt.toFox agglomerateIds: Array[Long] <- agglomerateService @@ -389,9 +390,10 @@ class DataSourceController @Inject()( } } - def update(token: Option[String], organizationId: String, datasetName: String): Action[DataSource] = + def update(organizationId: String, datasetName: String): Action[DataSource] = Action.async(validateJson[DataSource]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeDataSource(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.writeDataSource(DataSourceId(datasetName, organizationId))) { for { _ <- Fox.successful(()) dataSource <- dataSourceRepository.find(DataSourceId(datasetName, organizationId)).toFox ?~> Messages( @@ -402,10 +404,7 @@ class DataSourceController @Inject()( } // Stores a remote dataset in the database. - def add(token: Option[String], - organizationId: String, - datasetName: String, - folderId: Option[String]): Action[DataSource] = + def add(organizationId: String, datasetName: String, folderId: Option[String]): Action[DataSource] = Action.async(validateJson[DataSource]) { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDataSources) { for { @@ -433,21 +432,19 @@ class DataSourceController @Inject()( } } - def createOrganizationDirectory(token: Option[String], organizationId: String): Action[AnyContent] = Action.async { - implicit request => - accessTokenService.validateAccessFromTokenContextForSyncBlock(UserAccessRequest.administrateDataSources(organizationId)) { - val newOrganizationDirectory = new File(f"${dataSourceService.dataBaseDir}/$organizationId") - newOrganizationDirectory.mkdirs() - if (newOrganizationDirectory.isDirectory) - Ok - else - BadRequest - } + def createOrganizationDirectory(organizationId: String): Action[AnyContent] = Action.async { implicit request => + accessTokenService.validateAccessFromTokenContextForSyncBlock( + UserAccessRequest.administrateDataSources(organizationId)) { + val newOrganizationDirectory = new File(f"${dataSourceService.dataBaseDir}/$organizationId") + newOrganizationDirectory.mkdirs() + if (newOrganizationDirectory.isDirectory) + Ok + else + BadRequest + } } - def measureUsedStorage(token: Option[String], - organizationId: String, - datasetName: Option[String] = None): Action[AnyContent] = + def measureUsedStorage(organizationId: String, datasetName: Option[String] = None): Action[AnyContent] = Action.async { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDataSources(organizationId)) { @@ -465,10 +462,7 @@ class DataSourceController @Inject()( } } - def reload(token: Option[String], - organizationId: String, - datasetName: String, - layerName: Option[String] = None): Action[AnyContent] = + def reload(organizationId: String, datasetName: String, layerName: Option[String] = None): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDataSources(organizationId)) { val (closedAgglomerateFileHandleCount, clearedBucketProviderCount, removedChunksCount) = @@ -490,7 +484,7 @@ class DataSourceController @Inject()( } } - def deleteOnDisk(token: Option[String], organizationId: String, datasetName: String): Action[AnyContent] = + def deleteOnDisk(organizationId: String, datasetName: String): Action[AnyContent] = Action.async { implicit request => val dataSourceId = DataSourceId(datasetName, organizationId) accessTokenService.validateAccessFromTokenContext(UserAccessRequest.deleteDataSource(dataSourceId)) { @@ -504,9 +498,10 @@ class DataSourceController @Inject()( } } - def compose(token: Option[String]): Action[ComposeRequest] = + def compose(): Action[ComposeRequest] = Action.async(validateJson[ComposeRequest]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDataSources(request.body.organizationId)) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.administrateDataSources(request.body.organizationId)) { for { _ <- Fox.serialCombined(request.body.layers.map(_.datasetId).toList)(id => accessTokenService.assertUserAccess( @@ -517,12 +512,10 @@ class DataSourceController @Inject()( } } - def listConnectomeFiles(token: Option[String], - organizationId: String, - datasetName: String, - dataLayerName: String): Action[AnyContent] = + def listConnectomeFiles(organizationId: String, datasetName: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { val connectomeFileNames = connectomeFileService.exploreConnectomeFiles(organizationId, datasetName, dataLayerName) for { @@ -538,12 +531,12 @@ class DataSourceController @Inject()( } } - def getSynapsesForAgglomerates(token: Option[String], - organizationId: String, + def getSynapsesForAgglomerates(organizationId: String, datasetName: String, dataLayerName: String): Action[ByAgglomerateIdsRequest] = Action.async(validateJson[ByAgglomerateIdsRequest]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { meshFilePath <- Fox.successful( connectomeFileService @@ -553,13 +546,13 @@ class DataSourceController @Inject()( } } - def getSynapticPartnerForSynapses(token: Option[String], - organizationId: String, + def getSynapticPartnerForSynapses(organizationId: String, datasetName: String, dataLayerName: String, direction: String): Action[BySynapseIdsRequest] = Action.async(validateJson[BySynapseIdsRequest]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { meshFilePath <- Fox.successful( connectomeFileService @@ -571,12 +564,12 @@ class DataSourceController @Inject()( } } - def getSynapsePositions(token: Option[String], - organizationId: String, + def getSynapsePositions(organizationId: String, datasetName: String, dataLayerName: String): Action[BySynapseIdsRequest] = Action.async(validateJson[BySynapseIdsRequest]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { meshFilePath <- Fox.successful( connectomeFileService @@ -586,12 +579,10 @@ class DataSourceController @Inject()( } } - def getSynapseTypes(token: Option[String], - organizationId: String, - datasetName: String, - dataLayerName: String): Action[BySynapseIdsRequest] = + def getSynapseTypes(organizationId: String, datasetName: String, dataLayerName: String): Action[BySynapseIdsRequest] = Action.async(validateJson[BySynapseIdsRequest]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { meshFilePath <- Fox.successful( connectomeFileService @@ -601,12 +592,10 @@ class DataSourceController @Inject()( } } - def checkSegmentIndexFile(token: Option[String], - organizationId: String, - dataSetName: String, - dataLayerName: String): Action[AnyContent] = + def checkSegmentIndexFile(organizationId: String, dataSetName: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(dataSetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(dataSetName, organizationId))) { val segmentIndexFileOpt = segmentIndexFileService.getSegmentIndexFile(organizationId, dataSetName, dataLayerName).toOption Future.successful(Ok(Json.toJson(segmentIndexFileOpt.isDefined))) @@ -617,13 +606,13 @@ class DataSourceController @Inject()( * Query the segment index file for a single segment * @return List of bucketPositions as positions (not indices) of 32³ buckets in mag */ - def getSegmentIndex(token: Option[String], - organizationId: String, + def getSegmentIndex(organizationId: String, datasetName: String, dataLayerName: String, segmentId: String): Action[GetSegmentIndexParameters] = Action.async(validateJson[GetSegmentIndexParameters]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { segmentIds <- segmentIdsForAgglomerateIdIfNeeded( organizationId, @@ -655,12 +644,12 @@ class DataSourceController @Inject()( * Query the segment index file for multiple segments * @return List of bucketPositions as indices of 32³ buckets */ - def querySegmentIndex(token: Option[String], - organizationId: String, + def querySegmentIndex(organizationId: String, datasetName: String, dataLayerName: String): Action[GetMultipleSegmentIndexParameters] = Action.async(validateJson[GetMultipleSegmentIndexParameters]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { segmentIdsAndBucketPositions <- Fox.serialCombined(request.body.segmentIds) { segmentOrAgglomerateId => for { @@ -672,7 +661,7 @@ class DataSourceController @Inject()( request.body.editableMappingTracingId, segmentOrAgglomerateId, mappingNameForMeshFile = None, - omitMissing = true, // assume agglomerate ids not present in the mapping belong to user-brushed segments + omitMissing = true // assume agglomerate ids not present in the mapping belong to user-brushed segments ) fileMag <- segmentIndexFileService.readFileMag(organizationId, datasetName, dataLayerName) topLeftsNested: Seq[Array[Vec3Int]] <- Fox.serialCombined(segmentIds)(sId => @@ -687,12 +676,12 @@ class DataSourceController @Inject()( } } - def getSegmentVolume(token: Option[String], - organizationId: String, + def getSegmentVolume(organizationId: String, datasetName: String, dataLayerName: String): Action[SegmentStatisticsParameters] = Action.async(validateJson[SegmentStatisticsParameters]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { _ <- segmentIndexFileService.assertSegmentIndexFileExists(organizationId, datasetName, dataLayerName) volumes <- Fox.serialCombined(request.body.segmentIds) { segmentId => @@ -709,12 +698,12 @@ class DataSourceController @Inject()( } } - def getSegmentBoundingBox(token: Option[String], - organizationId: String, + def getSegmentBoundingBox(organizationId: String, datasetName: String, dataLayerName: String): Action[SegmentStatisticsParameters] = Action.async(validateJson[SegmentStatisticsParameters]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { _ <- segmentIndexFileService.assertSegmentIndexFileExists(organizationId, datasetName, dataLayerName) boxes <- Fox.serialCombined(request.body.segmentIds) { segmentId => @@ -730,9 +719,10 @@ class DataSourceController @Inject()( } // Called directly by wk side - def exploreRemoteDataset(token: Option[String]): Action[ExploreRemoteDatasetRequest] = + def exploreRemoteDataset(): Action[ExploreRemoteDatasetRequest] = Action.async(validateJson[ExploreRemoteDatasetRequest]) { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.administrateDataSources(request.body.organizationId)) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.administrateDataSources(request.body.organizationId)) { val reportMutable = ListBuffer[String]() val hasLocalFilesystemRequest = request.body.layerParameters.exists(param => new URI(param.remoteUri).getScheme == DataVaultService.schemeFile) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ExportsController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ExportsController.scala index efdac2ed607..e801528a4b3 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ExportsController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ExportsController.scala @@ -35,7 +35,7 @@ class ExportsController @Inject()(webknossosClient: DSRemoteWebknossosClient, override def allowRemoteOrigin: Boolean = true - def download(token: Option[String], jobId: String): Action[AnyContent] = Action.async { implicit request => + def download(jobId: String): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.downloadJobExport(jobId)) { for { exportProperties <- webknossosClient.getJobExportProperties(jobId) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala index a04a7d81c40..358c74eb442 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/ZarrStreamingController.scala @@ -51,12 +51,12 @@ class ZarrStreamingController @Inject()( * Uses the OME-NGFF standard (see https://ngff.openmicroscopy.org/latest/) */ def requestZAttrs( - token: Option[String], organizationId: String, datasetName: String, dataLayerName: String = "", ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -68,12 +68,12 @@ class ZarrStreamingController @Inject()( } def requestZarrJson( - token: Option[String], organizationId: String, datasetName: String, dataLayerName: String = "", ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { (dataSource, dataLayer) <- dataSourceRepository.getDataSourceAndDataLayer(organizationId, datasetName, @@ -88,9 +88,7 @@ class ZarrStreamingController @Inject()( } } - def zAttrsWithAnnotationPrivateLink(token: Option[String], - accessToken: String, - dataLayerName: String = ""): Action[AnyContent] = + def zAttrsWithAnnotationPrivateLink(accessToken: String, dataLayerName: String = ""): Action[AnyContent] = Action.async { implicit request => ifIsAnnotationLayerOrElse( accessToken, @@ -113,9 +111,7 @@ class ZarrStreamingController @Inject()( ) } - def zarrJsonWithAnnotationPrivateLink(token: Option[String], - accessToken: String, - dataLayerName: String = ""): Action[AnyContent] = + def zarrJsonWithAnnotationPrivateLink(accessToken: String, dataLayerName: String = ""): Action[AnyContent] = Action.async { implicit request => ifIsAnnotationLayerOrElse( accessToken, @@ -146,12 +142,12 @@ class ZarrStreamingController @Inject()( * Note that the result here is not necessarily equal to the file used in the underlying storage. */ def requestDataSource( - token: Option[String], organizationId: String, datasetName: String, zarrVersion: Int, ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { dataSource <- dataSourceRepository.findUsable(DataSourceId(datasetName, organizationId)).toFox ~> NOT_FOUND dataLayers = dataSource.dataLayers @@ -198,9 +194,7 @@ class ZarrStreamingController @Inject()( } } - def dataSourceWithAnnotationPrivateLink(token: Option[String], - accessToken: String, - zarrVersion: Int): Action[AnyContent] = + def dataSourceWithAnnotationPrivateLink(accessToken: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => for { annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken) ~> NOT_FOUND @@ -225,20 +219,19 @@ class ZarrStreamingController @Inject()( } def requestRawZarrCube( - token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mag: String, coordinates: String, ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { rawZarrCube(organizationId, datasetName, dataLayerName, mag, coordinates) } } - def rawZarrCubePrivateLink(token: Option[String], - accessToken: String, + def rawZarrCubePrivateLink(accessToken: String, dataLayerName: String, mag: String, coordinates: String): Action[AnyContent] = @@ -292,13 +285,14 @@ class ZarrStreamingController @Inject()( _ <- bool2Fox(notFoundIndices.isEmpty) ~> "zarr.chunkNotFound" ~> NOT_FOUND } yield Ok(data) - def requestZArray(token: Option[String], - organizationId: String, - datasetName: String, - dataLayerName: String, - mag: String, + def requestZArray( + organizationId: String, + datasetName: String, + dataLayerName: String, + mag: String, ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { zArray(organizationId, datasetName, dataLayerName, mag) } } @@ -313,13 +307,14 @@ class ZarrStreamingController @Inject()( zarrHeader = ZarrHeader.fromLayer(dataLayer, magParsed) } yield Ok(Json.toJson(zarrHeader)) - def requestZarrJsonForMag(token: Option[String], - organizationId: String, - datasetName: String, - dataLayerName: String, - mag: String, + def requestZarrJsonForMag( + organizationId: String, + datasetName: String, + dataLayerName: String, + mag: String, ): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { zarrJsonForMag(organizationId, datasetName, dataLayerName, mag) } } @@ -334,36 +329,32 @@ class ZarrStreamingController @Inject()( zarrHeader = Zarr3ArrayHeader.fromDataLayer(dataLayer) } yield Ok(Json.toJson(zarrHeader)) - def zArrayPrivateLink(token: Option[String], - accessToken: String, - dataLayerName: String, - mag: String): Action[AnyContent] = Action.async { implicit request => - ifIsAnnotationLayerOrElse( - accessToken, - dataLayerName, - ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantTokenContext) => - remoteTracingstoreClient - .getZArray(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl)(relevantTokenContext) - .map(z => Ok(Json.toJson(z))), - orElse = - annotationSource => zArray(annotationSource.organizationId, annotationSource.datasetName, dataLayerName, mag) - ) + def zArrayPrivateLink(accessToken: String, dataLayerName: String, mag: String): Action[AnyContent] = Action.async { + implicit request => + ifIsAnnotationLayerOrElse( + accessToken, + dataLayerName, + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantTokenContext) => + remoteTracingstoreClient + .getZArray(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl)(relevantTokenContext) + .map(z => Ok(Json.toJson(z))), + orElse = + annotationSource => zArray(annotationSource.organizationId, annotationSource.datasetName, dataLayerName, mag) + ) } - def zarrJsonPrivateLink(token: Option[String], - accessToken: String, - dataLayerName: String, - mag: String): Action[AnyContent] = Action.async { implicit request => - ifIsAnnotationLayerOrElse( - accessToken, - dataLayerName, - ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantTokenContext) => - remoteTracingstoreClient - .getZarrJson(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl)(relevantTokenContext) - .map(z => Ok(Json.toJson(z))), - orElse = annotationSource => - zarrJsonForMag(annotationSource.organizationId, annotationSource.datasetName, dataLayerName, mag) - ) + def zarrJsonPrivateLink(accessToken: String, dataLayerName: String, mag: String): Action[AnyContent] = Action.async { + implicit request => + ifIsAnnotationLayerOrElse( + accessToken, + dataLayerName, + ifIsAnnotationLayer = (annotationLayer, annotationSource, relevantTokenContext) => + remoteTracingstoreClient + .getZarrJson(annotationLayer.tracingId, mag, annotationSource.tracingStoreUrl)(relevantTokenContext) + .map(z => Ok(Json.toJson(z))), + orElse = annotationSource => + zarrJsonForMag(annotationSource.organizationId, annotationSource.datasetName, dataLayerName, mag) + ) } private def ifIsAnnotationLayerOrElse( @@ -382,14 +373,14 @@ class ZarrStreamingController @Inject()( } } yield result - def requestDataLayerMagFolderContents(token: Option[String], - organizationId: String, + def requestDataLayerMagFolderContents(organizationId: String, datasetName: String, dataLayerName: String, mag: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { dataLayerMagFolderContents(organizationId, datasetName, dataLayerName, mag, zarrVersion) } } @@ -413,8 +404,7 @@ class ZarrStreamingController @Inject()( additionalEntries )).withHeaders() - def dataLayerMagFolderContentsPrivateLink(token: Option[String], - accessToken: String, + def dataLayerMagFolderContentsPrivateLink(accessToken: String, dataLayerName: String, mag: String, zarrVersion: Int): Action[AnyContent] = @@ -445,12 +435,12 @@ class ZarrStreamingController @Inject()( ) } - def requestDataLayerFolderContents(token: Option[String], - organizationId: String, + def requestDataLayerFolderContents(organizationId: String, datasetName: String, dataLayerName: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { dataLayerFolderContents(organizationId, datasetName, dataLayerName, zarrVersion) } } @@ -474,8 +464,7 @@ class ZarrStreamingController @Inject()( additionalFiles ++ mags.map(_.toMagLiteral(allowScalar = true)) )).withHeaders() - def dataLayerFolderContentsPrivateLink(token: Option[String], - accessToken: String, + def dataLayerFolderContentsPrivateLink(accessToken: String, dataLayerName: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => @@ -502,12 +491,12 @@ class ZarrStreamingController @Inject()( ) } - def requestDataSourceFolderContents(token: Option[String], - organizationId: String, + def requestDataSourceFolderContents(organizationId: String, datasetName: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + accessTokenService.validateAccessFromTokenContext( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { for { dataSource <- dataSourceRepository.findUsable(DataSourceId(datasetName, organizationId)).toFox ?~> Messages( "dataSource.notFound") ~> NOT_FOUND @@ -523,9 +512,7 @@ class ZarrStreamingController @Inject()( } } - def dataSourceFolderContentsPrivateLink(token: Option[String], - accessToken: String, - zarrVersion: Int): Action[AnyContent] = + def dataSourceFolderContentsPrivateLink(accessToken: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => for { annotationSource <- remoteWebknossosClient.getAnnotationSource(accessToken) @@ -550,19 +537,17 @@ class ZarrStreamingController @Inject()( )) } - def requestZGroup(token: Option[String], - organizationId: String, - datasetName: String, - dataLayerName: String = ""): Action[AnyContent] = Action.async { implicit request => - accessTokenService.validateAccessFromTokenContextForSyncBlock( - UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { - Ok(zGroupJson) + def requestZGroup(organizationId: String, datasetName: String, dataLayerName: String = ""): Action[AnyContent] = + Action.async { implicit request => + accessTokenService.validateAccessFromTokenContextForSyncBlock( + UserAccessRequest.readDataSources(DataSourceId(datasetName, organizationId))) { + Ok(zGroupJson) + } } - } private def zGroupJson: JsValue = Json.toJson(NgffGroupHeader(zarr_format = 2)) - def zGroupPrivateLink(token: Option[String], accessToken: String, dataLayerName: String): Action[AnyContent] = + def zGroupPrivateLink(accessToken: String, dataLayerName: String): Action[AnyContent] = Action.async { implicit request => ifIsAnnotationLayerOrElse( accessToken, diff --git a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes index a4dca523cde..6d8ae4626fc 100644 --- a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes +++ b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes @@ -5,122 +5,122 @@ GET /health @com.scalableminds.webknossos.datastore.controllers.Application.health # Read image data -POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/data @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaWebknossos(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String) -POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/readData @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestRawCuboidPost(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String) -GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/data @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestRawCuboid(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, x: Int, y: Int, z: Int, width: Int, height: Int, depth: Int, mag: String, halfByte: Boolean ?= false, mappingName: Option[String]) -GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/thumbnail.jpg @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.thumbnailJpeg(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, x: Int, y: Int, z: Int, width: Int, height: Int, mag: String, mappingName: Option[String], intensityMin: Option[Double], intensityMax: Option[Double], color: Option[String], invertColor: Option[Boolean]) -GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/findData @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.findData(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String) -GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/histogram @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.histogram(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String) +POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/data @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaWebknossos(organizationId: String, datasetName: String, dataLayerName: String) +POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/readData @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestRawCuboidPost(organizationId: String, datasetName: String, dataLayerName: String) +GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/data @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestRawCuboid(organizationId: String, datasetName: String, dataLayerName: String, x: Int, y: Int, z: Int, width: Int, height: Int, depth: Int, mag: String, halfByte: Boolean ?= false, mappingName: Option[String]) +GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/thumbnail.jpg @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.thumbnailJpeg(organizationId: String, datasetName: String, dataLayerName: String, x: Int, y: Int, z: Int, width: Int, height: Int, mag: String, mappingName: Option[String], intensityMin: Option[Double], intensityMax: Option[Double], color: Option[String], invertColor: Option[Boolean]) +GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/findData @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.findData(organizationId: String, datasetName: String, dataLayerName: String) +GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/histogram @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.histogram(organizationId: String, datasetName: String, dataLayerName: String) # Knossos compatible routes -GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/mag:resolution/x:x/y:y/z:z/bucket.raw @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaKnossos(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, resolution: Int, x: Int, y: Int, z: Int, cubeSize: Int) +GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/mag:resolution/x:x/y:y/z:z/bucket.raw @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaKnossos(organizationId: String, datasetName: String, dataLayerName: String, resolution: Int, x: Int, y: Int, z: Int, cubeSize: Int) # Zarr2 compatible routes -GET /zarr/:organizationId/:datasetName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationId: String, datasetName: String, zarrVersion: Int = 2) -GET /zarr/:organizationId/:datasetName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationId: String, datasetName: String, zarrVersion: Int = 2) -GET /zarr/:organizationId/:datasetName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZGroup(token: Option[String], organizationId: String, datasetName: String, dataLayerName="") -GET /zarr/:organizationId/:datasetName/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSource(token: Option[String], organizationId: String, datasetName: String, zarrVersion: Int = 2) -GET /zarr/:organizationId/:datasetName/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, zarrVersion: Int = 2) -GET /zarr/:organizationId/:datasetName/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, zarrVersion: Int = 2) -GET /zarr/:organizationId/:datasetName/:dataLayerName/.zattrs @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZAttrs(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String) -GET /zarr/:organizationId/:datasetName/:dataLayerName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZGroup(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String) -GET /zarr/:organizationId/:datasetName/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) -GET /zarr/:organizationId/:datasetName/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) -GET /zarr/:organizationId/:datasetName/:dataLayerName/:mag/.zarray @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZArray(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mag: String) -GET /zarr/:organizationId/:datasetName/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestRawZarrCube(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mag: String, coordinates: String) - -GET /annotations/zarr/:accessTokenOrId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 2) -GET /annotations/zarr/:accessTokenOrId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 2) -GET /annotations/zarr/:accessTokenOrId/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zGroupPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName="") -GET /annotations/zarr/:accessTokenOrId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 2) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 2) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 2) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/.zattrs @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zAttrsWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zGroupPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/.zarray @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zArrayPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) -GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) +GET /zarr/:organizationId/:datasetName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(organizationId: String, datasetName: String, zarrVersion: Int = 2) +GET /zarr/:organizationId/:datasetName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(organizationId: String, datasetName: String, zarrVersion: Int = 2) +GET /zarr/:organizationId/:datasetName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZGroup(organizationId: String, datasetName: String, dataLayerName="") +GET /zarr/:organizationId/:datasetName/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSource(organizationId: String, datasetName: String, zarrVersion: Int = 2) +GET /zarr/:organizationId/:datasetName/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(organizationId: String, datasetName: String, dataLayerName: String, zarrVersion: Int = 2) +GET /zarr/:organizationId/:datasetName/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(organizationId: String, datasetName: String, dataLayerName: String, zarrVersion: Int = 2) +GET /zarr/:organizationId/:datasetName/:dataLayerName/.zattrs @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZAttrs(organizationId: String, datasetName: String, dataLayerName: String) +GET /zarr/:organizationId/:datasetName/:dataLayerName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZGroup(organizationId: String, datasetName: String, dataLayerName: String) +GET /zarr/:organizationId/:datasetName/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(organizationId: String, datasetName: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) +GET /zarr/:organizationId/:datasetName/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(organizationId: String, datasetName: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) +GET /zarr/:organizationId/:datasetName/:dataLayerName/:mag/.zarray @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZArray(organizationId: String, datasetName: String, dataLayerName: String, mag: String) +GET /zarr/:organizationId/:datasetName/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestRawZarrCube(organizationId: String, datasetName: String, dataLayerName: String, mag: String, coordinates: String) + +GET /annotations/zarr/:accessTokenOrId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(accessTokenOrId: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(accessTokenOrId: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zGroupPrivateLink(accessTokenOrId: String, dataLayerName="") +GET /annotations/zarr/:accessTokenOrId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceWithAnnotationPrivateLink(accessTokenOrId: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/.zattrs @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zAttrsWithAnnotationPrivateLink(accessTokenOrId: String, dataLayerName: String) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/.zgroup @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zGroupPrivateLink(accessTokenOrId: String, dataLayerName: String) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 2) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/.zarray @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zArrayPrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String) +GET /annotations/zarr/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) # Zarr3 compatible routes -GET /zarr3_experimental/:organizationId/:datasetName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationId: String, datasetName: String, zarrVersion: Int = 3) -GET /zarr3_experimental/:organizationId/:datasetName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(token: Option[String], organizationId: String, datasetName: String, zarrVersion: Int = 3) -GET /zarr3_experimental/:organizationId/:datasetName/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSource(token: Option[String], organizationId: String, datasetName: String, zarrVersion: Int = 3) -GET /zarr3_experimental/:organizationId/:datasetName/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, zarrVersion: Int = 3) -GET /zarr3_experimental/:organizationId/:datasetName/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, zarrVersion: Int = 3) -GET /zarr3_experimental/:organizationId/:datasetName/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZarrJson(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String) -GET /zarr3_experimental/:organizationId/:datasetName/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) -GET /zarr3_experimental/:organizationId/:datasetName/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) -GET /zarr3_experimental/:organizationId/:datasetName/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZarrJsonForMag(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mag: String) -GET /zarr3_experimental/:organizationId/:datasetName/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestRawZarrCube(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mag: String, coordinates: String) - -GET /annotations/zarr3_experimental/:accessTokenOrId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 3) -GET /annotations/zarr3_experimental/:accessTokenOrId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 3) -GET /annotations/zarr3_experimental/:accessTokenOrId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, zarrVersion: Int = 3) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 3) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 3) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zarrJsonWithAnnotationPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zarrJsonPrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String) -GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(token: Option[String], accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) +GET /zarr3_experimental/:organizationId/:datasetName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(organizationId: String, datasetName: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationId/:datasetName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSourceFolderContents(organizationId: String, datasetName: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationId/:datasetName/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataSource(organizationId: String, datasetName: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationId/:datasetName/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(organizationId: String, datasetName: String, dataLayerName: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationId/:datasetName/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerFolderContents(organizationId: String, datasetName: String, dataLayerName: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationId/:datasetName/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZarrJson(organizationId: String, datasetName: String, dataLayerName: String) +GET /zarr3_experimental/:organizationId/:datasetName/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(organizationId: String, datasetName: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationId/:datasetName/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestDataLayerMagFolderContents(organizationId: String, datasetName: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) +GET /zarr3_experimental/:organizationId/:datasetName/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestZarrJsonForMag(organizationId: String, datasetName: String, dataLayerName: String, mag: String) +GET /zarr3_experimental/:organizationId/:datasetName/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.requestRawZarrCube(organizationId: String, datasetName: String, dataLayerName: String, mag: String, coordinates: String) + +GET /annotations/zarr3_experimental/:accessTokenOrId @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(accessTokenOrId: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceFolderContentsPrivateLink(accessTokenOrId: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/datasource-properties.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataSourceWithAnnotationPrivateLink(accessTokenOrId: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerFolderContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zarrJsonWithAnnotationPrivateLink(accessTokenOrId: String, dataLayerName: String) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/ @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.dataLayerMagFolderContentsPrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String, zarrVersion: Int = 3) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/zarr.json @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.zarrJsonPrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String) +GET /annotations/zarr3_experimental/:accessTokenOrId/:dataLayerName/:mag/:coordinates @com.scalableminds.webknossos.datastore.controllers.ZarrStreamingController.rawZarrCubePrivateLink(accessTokenOrId: String, dataLayerName: String, mag: String, coordinates: String) # Segmentation mappings -GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/mappings/:mappingName @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.mappingJson(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mappingName: String) -GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/mappings @com.scalableminds.webknossos.datastore.controllers.DataSourceController.listMappings(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String) +GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/mappings/:mappingName @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.mappingJson(organizationId: String, datasetName: String, dataLayerName: String, mappingName: String) +GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/mappings @com.scalableminds.webknossos.datastore.controllers.DataSourceController.listMappings(organizationId: String, datasetName: String, dataLayerName: String) # Agglomerate files -GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/agglomerates @com.scalableminds.webknossos.datastore.controllers.DataSourceController.listAgglomerates(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String) -GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/agglomerates/:mappingName/skeleton/:agglomerateId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.generateAgglomerateSkeleton(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mappingName: String, agglomerateId: Long) -GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/agglomerates/:mappingName/agglomerateGraph/:agglomerateId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.agglomerateGraph(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mappingName: String, agglomerateId: Long) -GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/agglomerates/:mappingName/largestAgglomerateId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.largestAgglomerateId(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mappingName: String) -POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/agglomerates/:mappingName/agglomeratesForSegments @com.scalableminds.webknossos.datastore.controllers.DataSourceController.agglomerateIdsForSegmentIds(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mappingName: String) -GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/agglomerates/:mappingName/agglomeratesForAllSegments @com.scalableminds.webknossos.datastore.controllers.DataSourceController.agglomerateIdsForAllSegmentIds(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mappingName: String) -GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/agglomerates/:mappingName/positionForSegment @com.scalableminds.webknossos.datastore.controllers.DataSourceController.positionForSegmentViaAgglomerateFile(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, mappingName: String, segmentId: Long) +GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/agglomerates @com.scalableminds.webknossos.datastore.controllers.DataSourceController.listAgglomerates(organizationId: String, datasetName: String, dataLayerName: String) +GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/agglomerates/:mappingName/skeleton/:agglomerateId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.generateAgglomerateSkeleton(organizationId: String, datasetName: String, dataLayerName: String, mappingName: String, agglomerateId: Long) +GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/agglomerates/:mappingName/agglomerateGraph/:agglomerateId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.agglomerateGraph(organizationId: String, datasetName: String, dataLayerName: String, mappingName: String, agglomerateId: Long) +GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/agglomerates/:mappingName/largestAgglomerateId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.largestAgglomerateId(organizationId: String, datasetName: String, dataLayerName: String, mappingName: String) +POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/agglomerates/:mappingName/agglomeratesForSegments @com.scalableminds.webknossos.datastore.controllers.DataSourceController.agglomerateIdsForSegmentIds(organizationId: String, datasetName: String, dataLayerName: String, mappingName: String) +GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/agglomerates/:mappingName/agglomeratesForAllSegments @com.scalableminds.webknossos.datastore.controllers.DataSourceController.agglomerateIdsForAllSegmentIds(organizationId: String, datasetName: String, dataLayerName: String, mappingName: String) +GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/agglomerates/:mappingName/positionForSegment @com.scalableminds.webknossos.datastore.controllers.DataSourceController.positionForSegmentViaAgglomerateFile(organizationId: String, datasetName: String, dataLayerName: String, mappingName: String, segmentId: Long) # Mesh files -GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/meshes @com.scalableminds.webknossos.datastore.controllers.DSMeshController.listMeshFiles(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String) -POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/meshes/chunks @com.scalableminds.webknossos.datastore.controllers.DSMeshController.listMeshChunksForSegment(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, targetMappingName: Option[String], editableMappingTracingId: Option[String]) -POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/meshes/chunks/data @com.scalableminds.webknossos.datastore.controllers.DSMeshController.readMeshChunk(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String) -POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/meshes/fullMesh.stl @com.scalableminds.webknossos.datastore.controllers.DSMeshController.loadFullMeshStl(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String) +GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/meshes @com.scalableminds.webknossos.datastore.controllers.DSMeshController.listMeshFiles(organizationId: String, datasetName: String, dataLayerName: String) +POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/meshes/chunks @com.scalableminds.webknossos.datastore.controllers.DSMeshController.listMeshChunksForSegment(organizationId: String, datasetName: String, dataLayerName: String, targetMappingName: Option[String], editableMappingTracingId: Option[String]) +POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/meshes/chunks/data @com.scalableminds.webknossos.datastore.controllers.DSMeshController.readMeshChunk(organizationId: String, datasetName: String, dataLayerName: String) +POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/meshes/fullMesh.stl @com.scalableminds.webknossos.datastore.controllers.DSMeshController.loadFullMeshStl(organizationId: String, datasetName: String, dataLayerName: String) # Connectome files -GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/connectomes @com.scalableminds.webknossos.datastore.controllers.DataSourceController.listConnectomeFiles(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String) -POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/connectomes/synapses/positions @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSynapsePositions(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String) -POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/connectomes/synapses/types @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSynapseTypes(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String) -POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/connectomes/synapses/:direction @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSynapticPartnerForSynapses(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String, direction: String) -POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/connectomes/synapses @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSynapsesForAgglomerates(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String) +GET /datasets/:organizationId/:datasetName/layers/:dataLayerName/connectomes @com.scalableminds.webknossos.datastore.controllers.DataSourceController.listConnectomeFiles(organizationId: String, datasetName: String, dataLayerName: String) +POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/connectomes/synapses/positions @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSynapsePositions(organizationId: String, datasetName: String, dataLayerName: String) +POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/connectomes/synapses/types @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSynapseTypes(organizationId: String, datasetName: String, dataLayerName: String) +POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/connectomes/synapses/:direction @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSynapticPartnerForSynapses(organizationId: String, datasetName: String, dataLayerName: String, direction: String) +POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/connectomes/synapses @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSynapsesForAgglomerates(organizationId: String, datasetName: String, dataLayerName: String) # Ad-Hoc Meshing -POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/adHocMesh @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestAdHocMesh(token: Option[String], organizationId: String, datasetName: String, dataLayerName: String) +POST /datasets/:organizationId/:datasetName/layers/:dataLayerName/adHocMesh @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestAdHocMesh(organizationId: String, datasetName: String, dataLayerName: String) # Segment-Index files -GET /datasets/:organizationId/:dataSetName/layers/:dataLayerName/hasSegmentIndex @com.scalableminds.webknossos.datastore.controllers.DataSourceController.checkSegmentIndexFile(token: Option[String], organizationId: String, dataSetName: String, dataLayerName: String) -POST /datasets/:organizationId/:dataSetName/layers/:dataLayerName/segmentIndex @com.scalableminds.webknossos.datastore.controllers.DataSourceController.querySegmentIndex(token: Option[String], organizationId: String, dataSetName: String, dataLayerName: String) -POST /datasets/:organizationId/:dataSetName/layers/:dataLayerName/segmentIndex/:segmentId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSegmentIndex(token: Option[String], organizationId: String, dataSetName: String, dataLayerName: String, segmentId: String) -POST /datasets/:organizationId/:dataSetName/layers/:dataLayerName/segmentStatistics/volume @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSegmentVolume(token: Option[String], organizationId: String, dataSetName: String, dataLayerName: String) -POST /datasets/:organizationId/:dataSetName/layers/:dataLayerName/segmentStatistics/boundingBox @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSegmentBoundingBox(token: Option[String], organizationId: String, dataSetName: String, dataLayerName: String) +GET /datasets/:organizationId/:dataSetName/layers/:dataLayerName/hasSegmentIndex @com.scalableminds.webknossos.datastore.controllers.DataSourceController.checkSegmentIndexFile(organizationId: String, dataSetName: String, dataLayerName: String) +POST /datasets/:organizationId/:dataSetName/layers/:dataLayerName/segmentIndex @com.scalableminds.webknossos.datastore.controllers.DataSourceController.querySegmentIndex(organizationId: String, dataSetName: String, dataLayerName: String) +POST /datasets/:organizationId/:dataSetName/layers/:dataLayerName/segmentIndex/:segmentId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSegmentIndex(organizationId: String, dataSetName: String, dataLayerName: String, segmentId: String) +POST /datasets/:organizationId/:dataSetName/layers/:dataLayerName/segmentStatistics/volume @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSegmentVolume(organizationId: String, dataSetName: String, dataLayerName: String) +POST /datasets/:organizationId/:dataSetName/layers/:dataLayerName/segmentStatistics/boundingBox @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getSegmentBoundingBox(organizationId: String, dataSetName: String, dataLayerName: String) # DataSource management -GET /datasets @com.scalableminds.webknossos.datastore.controllers.DataSourceController.testChunk(token: Option[String], resumableChunkNumber: Int, resumableIdentifier: String) -POST /datasets @com.scalableminds.webknossos.datastore.controllers.DataSourceController.uploadChunk(token: Option[String]) -GET /datasets/getUnfinishedUploads @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getUnfinishedUploads(token: Option[String], organizationName: String) -POST /datasets/reserveUpload @com.scalableminds.webknossos.datastore.controllers.DataSourceController.reserveUpload(token: Option[String]) -POST /datasets/reserveManualUpload @com.scalableminds.webknossos.datastore.controllers.DataSourceController.reserveManualUpload(token: Option[String]) -POST /datasets/finishUpload @com.scalableminds.webknossos.datastore.controllers.DataSourceController.finishUpload(token: Option[String]) -POST /datasets/cancelUpload @com.scalableminds.webknossos.datastore.controllers.DataSourceController.cancelUpload(token: Option[String]) -GET /datasets/measureUsedStorage/:organizationId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.measureUsedStorage(token: Option[String], organizationId: String, datasetName: Option[String]) -GET /datasets/:organizationId/:datasetName/readInboxDataSource @com.scalableminds.webknossos.datastore.controllers.DataSourceController.readInboxDataSource(token: Option[String], organizationId: String, datasetName: String) -POST /datasets/:organizationId/:datasetName @com.scalableminds.webknossos.datastore.controllers.DataSourceController.update(token: Option[String], organizationId: String, datasetName: String) -PUT /datasets/:organizationId/:datasetName @com.scalableminds.webknossos.datastore.controllers.DataSourceController.add(token: Option[String], organizationId: String, datasetName: String, folderId: Option[String]) -DELETE /datasets/:organizationId/:datasetName/deleteOnDisk @com.scalableminds.webknossos.datastore.controllers.DataSourceController.deleteOnDisk(token: Option[String], organizationId: String, datasetName: String) -POST /datasets/compose @com.scalableminds.webknossos.datastore.controllers.DataSourceController.compose(token: Option[String]) -POST /datasets/exploreRemote @com.scalableminds.webknossos.datastore.controllers.DataSourceController.exploreRemoteDataset(token: Option[String]) +GET /datasets @com.scalableminds.webknossos.datastore.controllers.DataSourceController.testChunk(resumableChunkNumber: Int, resumableIdentifier: String) +POST /datasets @com.scalableminds.webknossos.datastore.controllers.DataSourceController.uploadChunk() +GET /datasets/getUnfinishedUploads @com.scalableminds.webknossos.datastore.controllers.DataSourceController.getUnfinishedUploads(organizationName: String) +POST /datasets/reserveUpload @com.scalableminds.webknossos.datastore.controllers.DataSourceController.reserveUpload() +POST /datasets/reserveManualUpload @com.scalableminds.webknossos.datastore.controllers.DataSourceController.reserveManualUpload() +POST /datasets/finishUpload @com.scalableminds.webknossos.datastore.controllers.DataSourceController.finishUpload() +POST /datasets/cancelUpload @com.scalableminds.webknossos.datastore.controllers.DataSourceController.cancelUpload() +GET /datasets/measureUsedStorage/:organizationId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.measureUsedStorage(organizationId: String, datasetName: Option[String]) +GET /datasets/:organizationId/:datasetName/readInboxDataSource @com.scalableminds.webknossos.datastore.controllers.DataSourceController.readInboxDataSource(organizationId: String, datasetName: String) +POST /datasets/:organizationId/:datasetName @com.scalableminds.webknossos.datastore.controllers.DataSourceController.update(organizationId: String, datasetName: String) +PUT /datasets/:organizationId/:datasetName @com.scalableminds.webknossos.datastore.controllers.DataSourceController.add(organizationId: String, datasetName: String, folderId: Option[String]) +DELETE /datasets/:organizationId/:datasetName/deleteOnDisk @com.scalableminds.webknossos.datastore.controllers.DataSourceController.deleteOnDisk(organizationId: String, datasetName: String) +POST /datasets/compose @com.scalableminds.webknossos.datastore.controllers.DataSourceController.compose() +POST /datasets/exploreRemote @com.scalableminds.webknossos.datastore.controllers.DataSourceController.exploreRemoteDataset() # Actions -POST /triggers/checkInboxBlocking @com.scalableminds.webknossos.datastore.controllers.DataSourceController.triggerInboxCheckBlocking(token: Option[String]) -POST /triggers/createOrganizationDirectory @com.scalableminds.webknossos.datastore.controllers.DataSourceController.createOrganizationDirectory(token: Option[String], organizationId: String) -POST /triggers/reload/:organizationId/:datasetName @com.scalableminds.webknossos.datastore.controllers.DataSourceController.reload(token: Option[String], organizationId: String, datasetName: String, layerName: Option[String]) +POST /triggers/checkInboxBlocking @com.scalableminds.webknossos.datastore.controllers.DataSourceController.triggerInboxCheckBlocking() +POST /triggers/createOrganizationDirectory @com.scalableminds.webknossos.datastore.controllers.DataSourceController.createOrganizationDirectory(organizationId: String) +POST /triggers/reload/:organizationId/:datasetName @com.scalableminds.webknossos.datastore.controllers.DataSourceController.reload(organizationId: String, datasetName: String, layerName: Option[String]) # Exports -GET /exports/:jobId/download @com.scalableminds.webknossos.datastore.controllers.ExportsController.download(token: Option[String], jobId: String) +GET /exports/:jobId/download @com.scalableminds.webknossos.datastore.controllers.ExportsController.download(jobId: String) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala index 711748b2dda..6ab093d315a 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala @@ -37,7 +37,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer bodyParsers: PlayBodyParsers) extends Controller { - def makeMappingEditable(token: Option[String], annotationId: String, tracingId: String): Action[AnyContent] = + def makeMappingEditable(annotationId: String, tracingId: String): Action[AnyContent] = Action.async { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { @@ -77,7 +77,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer /*// TODO integrate all of this into annotation update - def updateEditableMapping(token: Option[String], + def updateEditableMapping( annotationId: String, tracingId: String): Action[List[UpdateActionGroup]] = Action.async(validateJson[List[UpdateActionGroup]]) { implicit request => @@ -106,10 +106,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer } */ - def editableMappingInfo(token: Option[String], - annotationId: String, - tracingId: String, - version: Option[Long]): Action[AnyContent] = + def editableMappingInfo(annotationId: String, tracingId: String, version: Option[Long]): Action[AnyContent] = Action.async { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { @@ -123,31 +120,29 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer } } - def segmentIdsForAgglomerate(token: Option[String], - annotationId: String, - tracingId: String, - agglomerateId: Long): Action[AnyContent] = Action.async { implicit request => - log() { - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { - for { - tracing <- volumeTracingService.find(annotationId, tracingId) - _ <- editableMappingService.assertTracingHasEditableMapping(tracing) - remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - agglomerateGraphBox: Box[AgglomerateGraph] <- editableMappingService - .getAgglomerateGraphForId(tracingId, agglomerateId, remoteFallbackLayer) - .futureBox - segmentIds <- agglomerateGraphBox match { - case Full(agglomerateGraph) => Fox.successful(agglomerateGraph.segments) - case Empty => Fox.successful(List.empty) - case f: Failure => f.toFox - } - agglomerateIdIsPresent = agglomerateGraphBox.isDefined - } yield Ok(Json.toJson(EditableMappingSegmentListResult(segmentIds.toList, agglomerateIdIsPresent))) + def segmentIdsForAgglomerate(annotationId: String, tracingId: String, agglomerateId: Long): Action[AnyContent] = + Action.async { implicit request => + log() { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { + for { + tracing <- volumeTracingService.find(annotationId, tracingId) + _ <- editableMappingService.assertTracingHasEditableMapping(tracing) + remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) + agglomerateGraphBox: Box[AgglomerateGraph] <- editableMappingService + .getAgglomerateGraphForId(tracingId, agglomerateId, remoteFallbackLayer) + .futureBox + segmentIds <- agglomerateGraphBox match { + case Full(agglomerateGraph) => Fox.successful(agglomerateGraph.segments) + case Empty => Fox.successful(List.empty) + case f: Failure => f.toFox + } + agglomerateIdIsPresent = agglomerateGraphBox.isDefined + } yield Ok(Json.toJson(EditableMappingSegmentListResult(segmentIds.toList, agglomerateIdIsPresent))) + } } } - } - def agglomerateIdsForSegments(token: Option[String], annotationId: String, tracingId: String): Action[ListOfLong] = + def agglomerateIdsForSegments(annotationId: String, tracingId: String): Action[ListOfLong] = Action.async(validateProto[ListOfLong]) { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { @@ -171,7 +166,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer } } - def agglomerateGraphMinCut(token: Option[String], annotationId: String, tracingId: String): Action[MinCutParameters] = + def agglomerateGraphMinCut(annotationId: String, tracingId: String): Action[MinCutParameters] = Action.async(validateJson[MinCutParameters]) { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { @@ -185,9 +180,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer } } - def agglomerateGraphNeighbors(token: Option[String], - annotationId: String, - tracingId: String): Action[NeighborsParameters] = + def agglomerateGraphNeighbors(annotationId: String, tracingId: String): Action[NeighborsParameters] = Action.async(validateJson[NeighborsParameters]) { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala index 86feebacce9..79ebfb8138b 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala @@ -35,7 +35,7 @@ class SkeletonTracingController @Inject()(val tracingService: SkeletonTracingSer implicit def unpackMultiple(tracings: SkeletonTracings): List[Option[SkeletonTracing]] = tracings.tracings.toList.map(_.tracing) - def mergedFromContents(token: Option[String], persist: Boolean): Action[SkeletonTracings] = + def mergedFromContents(persist: Boolean): Action[SkeletonTracings] = Action.async(validateProto[SkeletonTracings]) { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { @@ -49,8 +49,7 @@ class SkeletonTracingController @Inject()(val tracingService: SkeletonTracingSer } } - def duplicate(token: Option[String], - annotationId: String, + def duplicate(annotationId: String, tracingId: String, version: Option[Long], fromTask: Option[Boolean], diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index e1ab8ec7070..f4fe393cc77 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -26,7 +26,7 @@ class TSAnnotationController @Inject()( extends Controller with KeyValueStoreImplicits { - def save(token: Option[String], annotationId: String): Action[AnnotationProto] = + def save(annotationId: String): Action[AnnotationProto] = Action.async(validateProto[AnnotationProto]) { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { @@ -39,7 +39,7 @@ class TSAnnotationController @Inject()( } } - def update(token: Option[String], annotationId: String): Action[List[UpdateActionGroup]] = + def update(annotationId: String): Action[List[UpdateActionGroup]] = Action.async(validateJson[List[UpdateActionGroup]]) { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { @@ -52,8 +52,7 @@ class TSAnnotationController @Inject()( } } - def updateActionLog(token: Option[String], - annotationId: String, + def updateActionLog(annotationId: String, newestVersion: Option[Long] = None, oldestVersion: Option[Long] = None): Action[AnyContent] = Action.async { implicit request => log() { @@ -65,29 +64,27 @@ class TSAnnotationController @Inject()( } } - def newestVersion(token: Option[String], annotationId: String): Action[AnyContent] = Action.async { - implicit request => - log() { - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readAnnotation(annotationId)) { - for { - newestVersion <- annotationService.currentMaterializableVersion(annotationId) - } yield JsonOk(Json.obj("version" -> newestVersion)) - } + def newestVersion(annotationId: String): Action[AnyContent] = Action.async { implicit request => + log() { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readAnnotation(annotationId)) { + for { + newestVersion <- annotationService.currentMaterializableVersion(annotationId) + } yield JsonOk(Json.obj("version" -> newestVersion)) } + } } - def updateActionStatistics(token: Option[String], tracingId: String): Action[AnyContent] = Action.async { - implicit request => - log() { - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { - for { - statistics <- annotationService.updateActionStatistics(tracingId) - } yield Ok(statistics) - } + def updateActionStatistics(tracingId: String): Action[AnyContent] = Action.async { implicit request => + log() { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { + for { + statistics <- annotationService.updateActionStatistics(tracingId) + } yield Ok(statistics) } + } } - def get(token: Option[String], annotationId: String, version: Option[Long]): Action[AnyContent] = + def get(annotationId: String, version: Option[Long]): Action[AnyContent] = Action.async { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala index 8bc7d999ea1..a8c1b04dd60 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala @@ -41,7 +41,7 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C override def allowRemoteOrigin: Boolean = true - def save(token: Option[String]): Action[T] = Action.async(validateProto[T]) { implicit request => + def save(): Action[T] = Action.async(validateProto[T]) { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { @@ -54,7 +54,7 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C } } - def saveMultiple(token: Option[String]): Action[Ts] = Action.async(validateProto[Ts]) { implicit request => + def saveMultiple(): Action[Ts] = Action.async(validateProto[Ts]) { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { @@ -70,7 +70,7 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C } } - def get(token: Option[String], annotationId: String, tracingId: String, version: Option[Long]): Action[AnyContent] = + def get(annotationId: String, tracingId: String, version: Option[Long]): Action[AnyContent] = Action.async { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { @@ -82,7 +82,7 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C } } - def getMultiple(token: Option[String]): Action[List[Option[TracingSelector]]] = + def getMultiple: Action[List[Option[TracingSelector]]] = Action.async(validateJson[List[Option[TracingSelector]]]) { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { @@ -95,7 +95,7 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C } } - def mergedFromIds(token: Option[String], persist: Boolean): Action[List[Option[TracingSelector]]] = + def mergedFromIds(persist: Boolean): Action[List[Option[TracingSelector]]] = Action.async(validateJson[List[Option[TracingSelector]]]) { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index ddba192d7bb..d0cd69fd7cf 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -78,8 +78,7 @@ class VolumeTracingController @Inject()( implicit def unpackMultiple(tracings: VolumeTracings): List[Option[VolumeTracing]] = tracings.tracings.toList.map(_.tracing) - def initialData(token: Option[String], - annotationId: String, + def initialData(annotationId: String, tracingId: String, minResolution: Option[Int], maxResolution: Option[Int]): Action[AnyContent] = @@ -101,7 +100,7 @@ class VolumeTracingController @Inject()( } } - def mergedFromContents(token: Option[String], persist: Boolean): Action[VolumeTracings] = + def mergedFromContents(persist: Boolean): Action[VolumeTracings] = Action.async(validateProto[VolumeTracings]) { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { @@ -120,7 +119,7 @@ class VolumeTracingController @Inject()( } } - def initialDataMultiple(token: Option[String], annotationId: String, tracingId: String): Action[AnyContent] = + def initialDataMultiple(annotationId: String, tracingId: String): Action[AnyContent] = Action.async { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { @@ -138,8 +137,7 @@ class VolumeTracingController @Inject()( } } - def allDataZip(token: Option[String], - annotationId: String, + def allDataZip(annotationId: String, tracingId: String, volumeDataZipFormat: String, version: Option[Long], @@ -166,7 +164,7 @@ class VolumeTracingController @Inject()( } } - def data(token: Option[String], annotationId: String, tracingId: String): Action[List[WebknossosDataRequest]] = + def data(annotationId: String, tracingId: String): Action[List[WebknossosDataRequest]] = Action.async(validateJson[List[WebknossosDataRequest]]) { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { @@ -186,8 +184,7 @@ class VolumeTracingController @Inject()( private def formatMissingBucketList(indices: List[Int]): String = "[" + indices.mkString(", ") + "]" - def duplicate(token: Option[String], - annotationId: String, + def duplicate(annotationId: String, tracingId: String, fromTask: Option[Boolean], minResolution: Option[Int], @@ -232,9 +229,7 @@ class VolumeTracingController @Inject()( } } - def importVolumeData(token: Option[String], - annotationId: String, - tracingId: String): Action[MultipartFormData[TemporaryFile]] = + def importVolumeData(annotationId: String, tracingId: String): Action[MultipartFormData[TemporaryFile]] = Action.async(parse.multipartFormData) { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeTracing(tracingId)) { @@ -252,10 +247,7 @@ class VolumeTracingController @Inject()( } } - def addSegmentIndex(token: Option[String], - annotationId: String, - tracingId: String, - dryRun: Boolean): Action[AnyContent] = + def addSegmentIndex(annotationId: String, tracingId: String, dryRun: Boolean): Action[AnyContent] = Action.async { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { @@ -279,8 +271,7 @@ class VolumeTracingController @Inject()( } } - def updateActionLog(token: Option[String], - tracingId: String, + def updateActionLog(tracingId: String, newestVersion: Option[Long] = None, oldestVersion: Option[Long] = None): Action[AnyContent] = Action.async { implicit request => log() { @@ -292,9 +283,7 @@ class VolumeTracingController @Inject()( } } - def requestAdHocMesh(token: Option[String], - annotationId: String, - tracingId: String): Action[WebknossosAdHocMeshRequest] = + def requestAdHocMesh(annotationId: String, tracingId: String): Action[WebknossosAdHocMeshRequest] = Action.async(validateJson[WebknossosAdHocMeshRequest]) { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { @@ -314,7 +303,7 @@ class VolumeTracingController @Inject()( } } - def loadFullMeshStl(token: Option[String], annotationId: String, tracingId: String): Action[FullMeshRequest] = + def loadFullMeshStl(annotationId: String, tracingId: String): Action[FullMeshRequest] = Action.async(validateJson[FullMeshRequest]) { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { @@ -329,21 +318,17 @@ class VolumeTracingController @Inject()( private def formatNeighborList(neighbors: List[Int]): String = "[" + neighbors.mkString(", ") + "]" - def findData(token: Option[String], annotationId: String, tracingId: String): Action[AnyContent] = Action.async { - implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { - for { - positionOpt <- tracingService.findData(annotationId, tracingId) - } yield { - Ok(Json.obj("position" -> positionOpt, "resolution" -> positionOpt.map(_ => Vec3Int.ones))) - } + def findData(annotationId: String, tracingId: String): Action[AnyContent] = Action.async { implicit request => + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { + for { + positionOpt <- tracingService.findData(annotationId, tracingId) + } yield { + Ok(Json.obj("position" -> positionOpt, "resolution" -> positionOpt.map(_ => Vec3Int.ones))) } + } } - def agglomerateSkeleton(token: Option[String], - annotationId: String, - tracingId: String, - agglomerateId: Long): Action[AnyContent] = + def agglomerateSkeleton(annotationId: String, tracingId: String, agglomerateId: Long): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { @@ -358,9 +343,7 @@ class VolumeTracingController @Inject()( } } - def getSegmentVolume(token: Option[String], - annotationId: String, - tracingId: String): Action[SegmentStatisticsParameters] = + def getSegmentVolume(annotationId: String, tracingId: String): Action[SegmentStatisticsParameters] = Action.async(validateJson[SegmentStatisticsParameters]) { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { @@ -378,9 +361,7 @@ class VolumeTracingController @Inject()( } } - def getSegmentBoundingBox(token: Option[String], - annotationId: String, - tracingId: String): Action[SegmentStatisticsParameters] = + def getSegmentBoundingBox(annotationId: String, tracingId: String): Action[SegmentStatisticsParameters] = Action.async(validateJson[SegmentStatisticsParameters]) { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { @@ -398,10 +379,7 @@ class VolumeTracingController @Inject()( } } - def getSegmentIndex(token: Option[String], - annotationId: String, - tracingId: String, - segmentId: Long): Action[GetSegmentIndexParameters] = + def getSegmentIndex(annotationId: String, tracingId: String, segmentId: Long): Action[GetSegmentIndexParameters] = Action.async(validateJson[GetSegmentIndexParameters]) { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala index 488c0c14c54..b1f58de065d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala @@ -57,10 +57,7 @@ class VolumeTracingZarrStreamingController @Inject()( override def defaultErrorCode: Int = NOT_FOUND - def volumeTracingFolderContent(token: Option[String], - annotationId: String, - tracingId: String, - zarrVersion: Int): Action[AnyContent] = + def volumeTracingFolderContent(annotationId: String, tracingId: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { @@ -79,10 +76,7 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def volumeTracingFolderContentJson(token: Option[String], - annotationId: String, - tracingId: String, - zarrVersion: Int): Action[AnyContent] = + def volumeTracingFolderContentJson(annotationId: String, tracingId: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { @@ -95,8 +89,7 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def volumeTracingMagFolderContent(token: Option[String], - annotationId: String, + def volumeTracingMagFolderContent(annotationId: String, tracingId: String, mag: String, zarrVersion: Int): Action[AnyContent] = @@ -119,8 +112,7 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def volumeTracingMagFolderContentJson(token: Option[String], - annotationId: String, + def volumeTracingMagFolderContentJson(annotationId: String, tracingId: String, mag: String, zarrVersion: Int): Action[AnyContent] = @@ -136,7 +128,7 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def zArray(token: Option[String], annotationId: String, tracingId: String, mag: String): Action[AnyContent] = + def zArray(annotationId: String, tracingId: String, mag: String): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { @@ -170,7 +162,7 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def zarrJsonForMag(token: Option[String], annotationId: String, tracingId: String, mag: String): Action[AnyContent] = + def zarrJsonForMag(annotationId: String, tracingId: String, mag: String): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { @@ -216,11 +208,10 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def zGroup(token: Option[String], annotationId: String, tracingId: String): Action[AnyContent] = Action.async { - implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { - Future(Ok(Json.toJson(NgffGroupHeader(zarr_format = 2)))) - } + def zGroup(annotationId: String, tracingId: String): Action[AnyContent] = Action.async { implicit request => + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { + Future(Ok(Json.toJson(NgffGroupHeader(zarr_format = 2)))) + } } /** @@ -229,7 +220,6 @@ class VolumeTracingZarrStreamingController @Inject()( * Used by zarr-streaming. */ def zAttrs( - token: Option[String], annotationId: String, tracingId: String, ): Action[AnyContent] = Action.async { implicit request => @@ -247,7 +237,6 @@ class VolumeTracingZarrStreamingController @Inject()( } def zarrJson( - token: Option[String], annotationId: String, tracingId: String, ): Action[AnyContent] = Action.async { implicit request => @@ -266,8 +255,7 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def zarrSource(token: Option[String], - annotationId: String, + def zarrSource(annotationId: String, tracingId: String, tracingName: Option[String], zarrVersion: Int): Action[AnyContent] = @@ -290,11 +278,7 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def rawZarrCube(token: Option[String], - annotationId: String, - tracingId: String, - mag: String, - coordinates: String): Action[AnyContent] = + def rawZarrCube(annotationId: String, tracingId: String, mag: String, coordinates: String): Action[AnyContent] = Action.async { implicit request => { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index f9a1e371dcd..8787b4d32e9 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -6,76 +6,76 @@ GET /health @com.scalableminds.webknossos.tracingstore.controllers.Application.health # Annotations (concerns AnnotationProto, not annotation info as stored in postgres) -POST /annotation/save @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.save(token: Option[String], annotationId: String) -GET /annotation/:annotationId @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.get(token: Option[String], annotationId: String, version: Option[Long]) -POST /annotation/:annotationId/update @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.update(token: Option[String], annotationId: String) -GET /annotation/:annotationId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionLog(token: Option[String], annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]) -GET /annotation/:annotationId/updateActionStatistics @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionStatistics(token: Option[String], annotationId: String) -GET /annotation/:annotationId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.newestVersion(token: Option[String], annotationId: String) +POST /annotation/save @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.save(annotationId: String) +GET /annotation/:annotationId @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.get(annotationId: String, version: Option[Long]) +POST /annotation/:annotationId/update @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.update(annotationId: String) +GET /annotation/:annotationId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionLog(annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]) +GET /annotation/:annotationId/updateActionStatistics @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionStatistics(annotationId: String) +GET /annotation/:annotationId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.newestVersion(annotationId: String) # Volume tracings -POST /volume/save @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.save(token: Option[String]) -POST /volume/:annotationId/:tracingId/initialData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.initialData(token: Option[String], annotationId: String, tracingId: String, minResolution: Option[Int], maxResolution: Option[Int]) -POST /volume/:annotationId/:tracingId/initialDataMultiple @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.initialDataMultiple(token: Option[String], annotationId: String, tracingId: String) -GET /volume/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.get(token: Option[String], annotationId: String, tracingId: String, version: Option[Long]) -GET /volume/:annotationId/:tracingId/allDataZip @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.allDataZip(token: Option[String], annotationId: String, tracingId: String, volumeDataZipFormat: String, version: Option[Long], voxelSize: Option[String], voxelSizeUnit: Option[String]) -POST /volume/:annotationId/:tracingId/data @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.data(token: Option[String], annotationId: String, tracingId: String) -POST /volume/:annotationId/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.duplicate(token: Option[String], annotationId: String, tracingId: String, fromTask: Option[Boolean], minResolution: Option[Int], maxResolution: Option[Int], downsample: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) -POST /volume/:annotationId/:tracingId/adHocMesh @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.requestAdHocMesh(token: Option[String], annotationId: String, tracingId: String) -POST /volume/:annotationId/:tracingId/fullMesh.stl @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.loadFullMeshStl(token: Option[String], annotationId: String, tracingId: String) -POST /volume/:annotationId/:tracingId/segmentIndex/:segmentId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentIndex(token: Option[String], annotationId: String, tracingId: String, segmentId: Long) -POST /volume/:annotationId/:tracingId/importVolumeData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.importVolumeData(token: Option[String], annotationId: String, tracingId: String) -POST /volume/:annotationId/:tracingId/addSegmentIndex @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.addSegmentIndex(token: Option[String], annotationId: String, tracingId: String, dryRun: Boolean) -GET /volume/:annotationId/:tracingId/findData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.findData(token: Option[String], annotationId: String, tracingId: String) -GET /volume/:annotationId/:tracingId/agglomerateSkeleton/:agglomerateId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.agglomerateSkeleton(token: Option[String], annotationId: String, tracingId: String, agglomerateId: Long) -POST /volume/:annotationId/:tracingId/segmentStatistics/volume @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentVolume(token: Option[String], annotationId: String, tracingId: String) -POST /volume/:annotationId/:tracingId/segmentStatistics/boundingBox @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentBoundingBox(token: Option[String], annotationId: String, tracingId: String) -POST /volume/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getMultiple(token: Option[String]) -POST /volume/mergedFromIds @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromIds(token: Option[String], persist: Boolean) -POST /volume/mergedFromContents @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromContents(token: Option[String], persist: Boolean) +POST /volume/save @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.save() +POST /volume/:annotationId/:tracingId/initialData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.initialData(annotationId: String, tracingId: String, minResolution: Option[Int], maxResolution: Option[Int]) +POST /volume/:annotationId/:tracingId/initialDataMultiple @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.initialDataMultiple(annotationId: String, tracingId: String) +GET /volume/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.get(annotationId: String, tracingId: String, version: Option[Long]) +GET /volume/:annotationId/:tracingId/allDataZip @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.allDataZip(annotationId: String, tracingId: String, volumeDataZipFormat: String, version: Option[Long], voxelSize: Option[String], voxelSizeUnit: Option[String]) +POST /volume/:annotationId/:tracingId/data @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.data(annotationId: String, tracingId: String) +POST /volume/:annotationId/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.duplicate(annotationId: String, tracingId: String, fromTask: Option[Boolean], minResolution: Option[Int], maxResolution: Option[Int], downsample: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) +POST /volume/:annotationId/:tracingId/adHocMesh @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.requestAdHocMesh(annotationId: String, tracingId: String) +POST /volume/:annotationId/:tracingId/fullMesh.stl @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.loadFullMeshStl(annotationId: String, tracingId: String) +POST /volume/:annotationId/:tracingId/segmentIndex/:segmentId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentIndex(annotationId: String, tracingId: String, segmentId: Long) +POST /volume/:annotationId/:tracingId/importVolumeData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.importVolumeData(annotationId: String, tracingId: String) +POST /volume/:annotationId/:tracingId/addSegmentIndex @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.addSegmentIndex(annotationId: String, tracingId: String, dryRun: Boolean) +GET /volume/:annotationId/:tracingId/findData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.findData(annotationId: String, tracingId: String) +GET /volume/:annotationId/:tracingId/agglomerateSkeleton/:agglomerateId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.agglomerateSkeleton(annotationId: String, tracingId: String, agglomerateId: Long) +POST /volume/:annotationId/:tracingId/segmentStatistics/volume @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentVolume(annotationId: String, tracingId: String) +POST /volume/:annotationId/:tracingId/segmentStatistics/boundingBox @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentBoundingBox(annotationId: String, tracingId: String) +POST /volume/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getMultiple +POST /volume/mergedFromIds @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromIds(persist: Boolean) +POST /volume/mergedFromContents @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromContents(persist: Boolean) # Editable Mappings # todo adapt frontend to mapping route prefix -POST /mapping/:annotationId/:tracingId/makeMappingEditable @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.makeMappingEditable(token: Option[String], annotationId: String, tracingId: String) -GET /mapping/:annotationId/:tracingId/info @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.editableMappingInfo(token: Option[String], annotationId: String, tracingId: String, version: Option[Long]) -GET /mapping/:annotationId/:tracingId/segmentsForAgglomerate @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.segmentIdsForAgglomerate(token: Option[String], annotationId: String, tracingId: String, agglomerateId: Long) -POST /mapping/:annotationId/:tracingId/agglomeratesForSegments @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateIdsForSegments(token: Option[String], annotationId: String, tracingId: String) +POST /mapping/:annotationId/:tracingId/makeMappingEditable @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.makeMappingEditable(annotationId: String, tracingId: String) +GET /mapping/:annotationId/:tracingId/info @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.editableMappingInfo(annotationId: String, tracingId: String, version: Option[Long]) +GET /mapping/:annotationId/:tracingId/segmentsForAgglomerate @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.segmentIdsForAgglomerate(annotationId: String, tracingId: String, agglomerateId: Long) +POST /mapping/:annotationId/:tracingId/agglomeratesForSegments @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateIdsForSegments(annotationId: String, tracingId: String) # todo adapt frontend to mapping route prefix -POST /mapping/:annotationId/:tracingId/agglomerateGraphMinCut @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateGraphMinCut(token: Option[String], annotationId: String, tracingId: String) +POST /mapping/:annotationId/:tracingId/agglomerateGraphMinCut @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateGraphMinCut(annotationId: String, tracingId: String) # todo adapt frontend to mapping route prefix -POST /mapping/:annotationId/:tracingId/agglomerateGraphNeighbors @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateGraphNeighbors(token: Option[String], annotationId: String, tracingId: String) +POST /mapping/:annotationId/:tracingId/agglomerateGraphNeighbors @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateGraphNeighbors(annotationId: String, tracingId: String) # Zarr endpoints for volume annotations # Zarr version 2 -GET /volume/zarr/json/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(token: Option[String], annotationId: String, tracingId: String, zarrVersion: Int = 2) -GET /volume/zarr/json/:annotationId/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(token: Option[String], annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 2) -GET /volume/zarr/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], annotationId: String, tracingId: String, zarrVersion: Int = 2) -GET /volume/zarr/:annotationId/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], annotationId: String, tracingId: String, zarrVersion: Int = 2) -GET /volume/zarr/:annotationId/:tracingId/.zgroup @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zGroup(token: Option[String], annotationId: String, tracingId: String) -GET /volume/zarr/:annotationId/:tracingId/.zattrs @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zAttrs(token: Option[String], annotationId: String, tracingId: String) -GET /volume/zarr/:annotationId/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(token: Option[String], annotationId: String, tracingId: String, tracingName: Option[String], zarrVersion: Int = 2) -GET /volume/zarr/:annotationId/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 2) -GET /volume/zarr/:annotationId/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 2) -GET /volume/zarr/:annotationId/:tracingId/:mag/.zarray @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zArray(token: Option[String], annotationId: String, tracingId: String, mag: String) -GET /volume/zarr/:annotationId/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], annotationId: String, tracingId: String, mag: String, coordinates: String) +GET /volume/zarr/json/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(annotationId: String, tracingId: String, zarrVersion: Int = 2) +GET /volume/zarr/json/:annotationId/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 2) +GET /volume/zarr/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(annotationId: String, tracingId: String, zarrVersion: Int = 2) +GET /volume/zarr/:annotationId/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(annotationId: String, tracingId: String, zarrVersion: Int = 2) +GET /volume/zarr/:annotationId/:tracingId/.zgroup @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zGroup(annotationId: String, tracingId: String) +GET /volume/zarr/:annotationId/:tracingId/.zattrs @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zAttrs(annotationId: String, tracingId: String) +GET /volume/zarr/:annotationId/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(annotationId: String, tracingId: String, tracingName: Option[String], zarrVersion: Int = 2) +GET /volume/zarr/:annotationId/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 2) +GET /volume/zarr/:annotationId/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 2) +GET /volume/zarr/:annotationId/:tracingId/:mag/.zarray @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zArray(annotationId: String, tracingId: String, mag: String) +GET /volume/zarr/:annotationId/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(annotationId: String, tracingId: String, mag: String, coordinates: String) # Zarr version 3 -GET /volume/zarr3_experimental/json/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(token: Option[String], annotationId: String, tracingId: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/json/:annotationId/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(token: Option[String], annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], annotationId: String, tracingId: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:annotationId/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(token: Option[String], annotationId: String, tracingId: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:annotationId/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(token: Option[String], annotationId: String, tracingId: String, tracingName: Option[String], zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:annotationId/:tracingId/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJson(token: Option[String], annotationId: String, tracingId: String) -GET /volume/zarr3_experimental/:annotationId/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:annotationId/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(token: Option[String], annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:annotationId/:tracingId/:mag/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJsonForMag(token: Option[String], annotationId: String, tracingId: String, mag: String) -GET /volume/zarr3_experimental/:annotationId/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(token: Option[String], annotationId: String, tracingId: String, mag: String, coordinates: String) +GET /volume/zarr3_experimental/json/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(annotationId: String, tracingId: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/json/:annotationId/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(annotationId: String, tracingId: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:annotationId/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(annotationId: String, tracingId: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:annotationId/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(annotationId: String, tracingId: String, tracingName: Option[String], zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:annotationId/:tracingId/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJson(annotationId: String, tracingId: String) +GET /volume/zarr3_experimental/:annotationId/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:annotationId/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:annotationId/:tracingId/:mag/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJsonForMag(annotationId: String, tracingId: String, mag: String) +GET /volume/zarr3_experimental/:annotationId/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(annotationId: String, tracingId: String, mag: String, coordinates: String) # Skeleton tracings -POST /skeleton/save @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.save(token: Option[String]) -POST /skeleton/saveMultiple @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.saveMultiple(token: Option[String]) -POST /skeleton/mergedFromContents @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.mergedFromContents(token: Option[String], persist: Boolean) -POST /skeleton/mergedFromIds @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.mergedFromIds(token: Option[String], persist: Boolean) -GET /skeleton/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.get(token: Option[String], annotationId: String, tracingId: String, version: Option[Long]) -POST /skeleton/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.getMultiple(token: Option[String]) -POST /skeleton/:annotationId/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.duplicate(token: Option[String], annotationId: String, tracingId: String, version: Option[Long], fromTask: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) +POST /skeleton/save @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.save() +POST /skeleton/saveMultiple @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.saveMultiple() +POST /skeleton/mergedFromContents @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.mergedFromContents(persist: Boolean) +POST /skeleton/mergedFromIds @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.mergedFromIds(persist: Boolean) +GET /skeleton/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.get(annotationId: String, tracingId: String, version: Option[Long]) +POST /skeleton/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.getMultiple +POST /skeleton/:annotationId/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.duplicate(annotationId: String, tracingId: String, version: Option[Long], fromTask: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) From 2dc833e1795987fa0d81068d8b1582c70bb2fefe Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 18 Sep 2024 11:58:27 +0200 Subject: [PATCH 067/150] apply editable mapping updates --- .../tracingstore/annotation/AnnotationWithTracings.scala | 7 +++++-- .../tracingstore/annotation/TSAnnotationService.scala | 1 + .../editablemapping/EditableMappingUpdateActions.scala | 6 ++++-- .../tracings/editablemapping/EditableMappingUpdater.scala | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index ced731c2e80..a0ba363d1cb 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -1,6 +1,7 @@ package com.scalableminds.webknossos.tracingstore.annotation import com.scalableminds.util.tools.Fox +import com.scalableminds.util.tools.Fox.{box2Fox, option2Fox} import com.scalableminds.webknossos.datastore.Annotation.{AnnotationLayerProto, AnnotationProto} import com.scalableminds.webknossos.datastore.EditableMappingInfo.EditableMappingInfo import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing @@ -119,6 +120,8 @@ case class AnnotationWithTracings( def applyEditableMappingAction(a: EditableMappingUpdateAction)( implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { - updater <- getEditableMappingUpdater("tracingId") // TODO editable mapping update actions need tracing id - } yield this // TODO + updater: EditableMappingUpdater <- getEditableMappingUpdater(a.actionTracingId).toFox // TODO editable mapping update actions need tracing id + info <- getEditableMappingInfo(a.actionTracingId).toFox + _ <- updater.applyOneUpdate(info, a) + } yield this // TODO replace info } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index eb422d9fff1..13e0eab006a 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -281,6 +281,7 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl else { for { updated <- updateIter(Some(annotation), updates) + // TODO flush editable mapping updaters } yield updated.withVersion(targetVersion) } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala index 7f037238097..2bbbda3643a 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala @@ -1,11 +1,11 @@ package com.scalableminds.webknossos.tracingstore.tracings.editablemapping import com.scalableminds.util.geometry.Vec3Int -import com.scalableminds.webknossos.tracingstore.annotation.UpdateAction +import com.scalableminds.webknossos.tracingstore.annotation.{LayerUpdateAction, UpdateAction} import play.api.libs.json.Format.GenericFormat import play.api.libs.json._ -trait EditableMappingUpdateAction extends UpdateAction +trait EditableMappingUpdateAction extends LayerUpdateAction // we switched from positions to segment ids in https://github.com/scalableminds/webknossos/pull/7742. // Both are now optional to support applying old update actions stored in the db. @@ -15,6 +15,7 @@ case class SplitAgglomerateUpdateAction(agglomerateId: Long, segmentId1: Option[Long], segmentId2: Option[Long], mag: Vec3Int, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -40,6 +41,7 @@ case class MergeAgglomerateUpdateAction(agglomerateId1: Long, segmentId1: Option[Long], segmentId2: Option[Long], mag: Vec3Int, + actionTracingId: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index d0baa685453..8334d5138e1 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -110,7 +110,7 @@ class EditableMappingUpdater( mappingFox } - private def applyOneUpdate(mapping: EditableMappingInfo, update: UpdateAction)( + def applyOneUpdate(mapping: EditableMappingInfo, update: UpdateAction)( implicit ec: ExecutionContext): Fox[EditableMappingInfo] = update match { case splitAction: SplitAgglomerateUpdateAction => From 0dda12ad26c0ba7778138fb75f31b9caa1a1639f Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 19 Sep 2024 11:47:42 +0200 Subject: [PATCH 068/150] update editable mapping info in AnnotationWithTracings --- .../annotation/AnnotationWithTracings.scala | 40 ++++++------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index a0ba363d1cb..bf5cc93b7fc 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -59,54 +59,38 @@ case class AnnotationWithTracings( def version: Long = annotation.version def addTracing(a: AddLayerAnnotationUpdateAction): AnnotationWithTracings = - AnnotationWithTracings( - annotation.copy( + this.copy( + annotation = annotation.copy( layers = annotation.layers :+ AnnotationLayerProto( a.tracingId, a.layerParameters.name.getOrElse(AnnotationLayer.defaultNameForType(a.layerParameters.typ)), `type` = AnnotationLayerType.toProto(a.layerParameters.typ) - )), - tracingsById, - editableMappingsByTracingId + )) ) def deleteTracing(a: DeleteLayerAnnotationUpdateAction): AnnotationWithTracings = - AnnotationWithTracings(annotation.copy(layers = annotation.layers.filter(_.tracingId != a.tracingId)), - tracingsById, - editableMappingsByTracingId) + this.copy(annotation = annotation.copy(layers = annotation.layers.filter(_.tracingId != a.tracingId))) def updateLayerMetadata(a: UpdateLayerMetadataAnnotationUpdateAction): AnnotationWithTracings = - AnnotationWithTracings(annotation.copy(layers = annotation.layers.map(l => - if (l.tracingId == a.tracingId) l.copy(name = a.layerName) else l)), - tracingsById, - editableMappingsByTracingId) + this.copy(annotation = annotation.copy(layers = annotation.layers.map(l => + if (l.tracingId == a.tracingId) l.copy(name = a.layerName) else l))) def updateMetadata(a: UpdateMetadataAnnotationUpdateAction): AnnotationWithTracings = - AnnotationWithTracings(annotation.copy(name = a.name, description = a.description), - tracingsById, - editableMappingsByTracingId) - - def incrementVersion: AnnotationWithTracings = - AnnotationWithTracings(annotation.copy(version = annotation.version + 1L), - tracingsById, - editableMappingsByTracingId) + this.copy(annotation = annotation.copy(name = a.name, description = a.description)) def withVersion(newVersion: Long): AnnotationWithTracings = { val tracingsUpdated = tracingsById.view.mapValues { case Left(t: SkeletonTracing) => Left(t.withVersion(newVersion)) case Right(t: VolumeTracing) => Right(t.withVersion(newVersion)) } - AnnotationWithTracings(annotation.copy(version = newVersion), tracingsUpdated.toMap, editableMappingsByTracingId) + this.copy(annotation = annotation.copy(version = newVersion), tracingsById = tracingsUpdated.toMap) } def applySkeletonAction(a: SkeletonUpdateAction)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { skeletonTracing <- getSkeleton(a.actionTracingId) updated = a.applyOn(skeletonTracing) - } yield - AnnotationWithTracings(annotation, - tracingsById.updated(a.actionTracingId, Left(updated)), - editableMappingsByTracingId) + } yield this.copy(tracingsById = tracingsById.updated(a.actionTracingId, Left(updated))) def applyVolumeAction(a: ApplyableVolumeUpdateAction)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { @@ -122,6 +106,8 @@ case class AnnotationWithTracings( for { updater: EditableMappingUpdater <- getEditableMappingUpdater(a.actionTracingId).toFox // TODO editable mapping update actions need tracing id info <- getEditableMappingInfo(a.actionTracingId).toFox - _ <- updater.applyOneUpdate(info, a) - } yield this // TODO replace info + updated <- updater.applyOneUpdate(info, a) + } yield + this.copy( + editableMappingsByTracingId = editableMappingsByTracingId.updated(a.actionTracingId, (updated, updater))) } From 520d54b4f66acbc51e431e401cc5e25e8b443816 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 19 Sep 2024 13:57:41 +0200 Subject: [PATCH 069/150] iterate on editable mappings --- frontend/javascripts/admin/admin_rest_api.ts | 8 +++++--- .../oxalis/model/sagas/proofread_saga.ts | 8 ++++++-- .../webknossos/datastore/rpc/RPCRequest.scala | 3 ++- .../annotation/TSAnnotationService.scala | 20 +++++++++++-------- .../EditableMappingController.scala | 15 +++++++------- .../EditableMappingService.scala | 10 ++++++---- ...alableminds.webknossos.tracingstore.routes | 3 --- 7 files changed, 39 insertions(+), 28 deletions(-) diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts index e341bf573a1..75faa599ae8 100644 --- a/frontend/javascripts/admin/admin_rest_api.ts +++ b/frontend/javascripts/admin/admin_rest_api.ts @@ -1619,7 +1619,7 @@ export function makeMappingEditable( ): Promise { return doWithToken((token) => Request.receiveJSON( - `${tracingStoreUrl}/tracings/volume/${annotationId}/${tracingId}/makeMappingEditable?token=${token}`, + `${tracingStoreUrl}/tracings/mapping/${annotationId}/${tracingId}/makeMappingEditable?token=${token}`, { method: "POST", }, @@ -2310,6 +2310,7 @@ type MinCutTargetEdge = { }; export async function getEdgesForAgglomerateMinCut( tracingStoreUrl: string, + annotationId: string, tracingId: string, segmentsInfo: { segmentId1: NumberLike; @@ -2321,7 +2322,7 @@ export async function getEdgesForAgglomerateMinCut( ): Promise> { return doWithToken((token) => Request.sendJSONReceiveJSON( - `${tracingStoreUrl}/tracings/volume/${tracingId}/agglomerateGraphMinCut?token=${token}`, + `${tracingStoreUrl}/tracings/mapping/${annotationId}/${tracingId}/agglomerateGraphMinCut?token=${token}`, { data: { ...segmentsInfo, @@ -2343,6 +2344,7 @@ export type NeighborInfo = { export async function getNeighborsForAgglomerateNode( tracingStoreUrl: string, tracingId: string, + annotationId: string, segmentInfo: { segmentId: NumberLike; mag: Vector3; @@ -2352,7 +2354,7 @@ export async function getNeighborsForAgglomerateNode( ): Promise { return doWithToken((token) => Request.sendJSONReceiveJSON( - `${tracingStoreUrl}/tracings/volume/${tracingId}/agglomerateGraphNeighbors?token=${token}`, + `${tracingStoreUrl}/tracings/mapping/${annotationId}/${tracingId}/agglomerateGraphNeighbors?token=${token}`, { data: { ...segmentInfo, diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts index e4e911bf1c6..b9776143517 100644 --- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts @@ -284,10 +284,10 @@ function* createEditableMapping(): Saga { ); // The server increments the volume tracing's version by 1 when switching the mapping to an editable one yield* put(setVersionNumberAction(upToDateVolumeTracing.version + 1, "volume", volumeTracingId)); - yield* put(setMappingNameAction(layerName, serverEditableMapping.tracingId, "HDF5")); + yield* put(setMappingNameAction(layerName, volumeTracingId, "HDF5")); yield* put(setHasEditableMappingAction()); yield* put(initializeEditableMappingAction(serverEditableMapping)); - return serverEditableMapping.tracingId; + return volumeTracingId; } function* ensureHdf5MappingIsEnabled(layerName: string): Saga { @@ -546,6 +546,7 @@ function* performMinCut( } const tracingStoreUrl = yield* select((state) => state.tracing.tracingStore.url); + const annotationId = yield* select((state) => state.tracing.annotationId); const segmentsInfo = { segmentId1: sourceSegmentId, segmentId2: targetSegmentId, @@ -557,6 +558,7 @@ function* performMinCut( const edgesToRemove = yield* call( getEdgesForAgglomerateMinCut, tracingStoreUrl, + annotationId, volumeTracingId, segmentsInfo, ); @@ -607,6 +609,7 @@ function* performCutFromNeighbors( { didCancel: false; neighborInfo: NeighborInfo } | { didCancel: true; neighborInfo?: null } > { const tracingStoreUrl = yield* select((state) => state.tracing.tracingStore.url); + const annotationId = yield* select((state) => state.tracing.annotationId); const segmentsInfo = { segmentId, mag: agglomerateFileMag, @@ -617,6 +620,7 @@ function* performCutFromNeighbors( const neighborInfo = yield* call( getNeighborsForAgglomerateNode, tracingStoreUrl, + annotationId, volumeTracingId, segmentsInfo, ); diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala index d26675b7e6d..4bd9e2872eb 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala @@ -11,6 +11,7 @@ import play.api.libs.ws._ import scalapb.{GeneratedMessage, GeneratedMessageCompanion} import java.io.File +import java.nio.charset.StandardCharsets import scala.concurrent.ExecutionContext import scala.concurrent.duration._ @@ -203,7 +204,7 @@ class RPCRequest(val id: Int, val url: String, wsClient: WSClient)(implicit ec: Full(result) } else { val errorMsg = s"Unsuccessful WS request to $url (ID: $id)." + - s"Status: ${result.status}. Response: ${result.bodyAsBytes.map(_.toChar).mkString.take(2000)}" + s"Status: ${result.status}. Response: ${new String(result.bodyAsBytes.toArray, StandardCharsets.UTF_8).take(2000)}" logger.error(errorMsg) Failure(errorMsg.take(400)) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 13e0eab006a..0cbe103f4d9 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -3,7 +3,7 @@ package com.scalableminds.webknossos.tracingstore.annotation import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox -import com.scalableminds.util.tools.Fox.option2Fox +import com.scalableminds.util.tools.Fox.{box2Fox, option2Fox} import com.scalableminds.webknossos.datastore.Annotation.AnnotationProto import com.scalableminds.webknossos.datastore.EditableMappingInfo.EditableMappingInfo import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing @@ -23,6 +23,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ ApplyableVolumeUpdateAction, BucketMutatingVolumeUpdateAction, + UpdateMappingNameVolumeAction, VolumeUpdateAction } import com.scalableminds.webknossos.tracingstore.tracings.{ @@ -94,13 +95,16 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl Fox.successful(annotationWithTracings.updateMetadata(a)) case a: SkeletonUpdateAction => annotationWithTracings.applySkeletonAction(a) ?~> "applySkeletonAction.failed" + case a: UpdateMappingNameVolumeAction if a.isEditable.contains(true) => + + TODO in case mapping is made editable, add it to the AnnotationWithTracings object here + case a: ApplyableVolumeUpdateAction => annotationWithTracings.applyVolumeAction(a) case a: EditableMappingUpdateAction => annotationWithTracings.applyEditableMappingAction(a) case _: BucketMutatingVolumeUpdateAction => Fox.successful(annotationWithTracings) // No-op, as bucket-mutating actions are performed eagerly, so not here. - // TODO make Mapping Editable // Note: UpdateBucketVolumeActions are not handled here, but instead eagerly on saving. case _ => Fox.failure(s"Received unsupported AnnotationUpdateAction action ${Json.toJson(updateAction)}") } @@ -152,8 +156,8 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl implicit ec: ExecutionContext, tc: TokenContext): Fox[EditableMappingInfo] = for { - annotation <- getWithTracings(annotationId, version, List(tracingId), List.empty) - tracing <- annotation.getEditableMappingInfo(tracingId) + annotation <- getWithTracings(annotationId, version, List.empty, List(tracingId)) + tracing <- annotation.getEditableMappingInfo(tracingId) ?~> "getEditableMapping.failed" } yield tracing private def applyPendingUpdates(annotation: AnnotationProto, @@ -183,6 +187,7 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl currentMaterializedVersion: Long, targetVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext) = { val volumeIdsWithEditableMapping = annotationWithTracings.volumesIdsThatHaveEditableMapping + logger.info(s"fetching editable mappings ${volumeIdsWithEditableMapping.mkString(",")}") // TODO intersect with editable mapping updates? for { editableMappingInfos <- Fox.serialCombined(volumeIdsWithEditableMapping) { volumeTracingId => @@ -236,9 +241,8 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl case u: VolumeUpdateAction => Some(u.actionTracingId) case _ => None } ++ requestedVolumeTracingIds).distinct - // TODO fetch editable mappings + instantiate editableMappingUpdaters/buffers if there are updates for them - val editableMappingsMap: Map[String, (EditableMappingInfo, EditableMappingUpdater)] = Map.empty - logger.info(s"fetching volumes ${volumeTracingIds} and skeletons $skeletonTracingIds") + + logger.info(s"fetching volumes $volumeTracingIds and skeletons $skeletonTracingIds") for { skeletonTracings <- Fox.serialCombined(skeletonTracingIds)( id => @@ -255,7 +259,7 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl volumeTracingsMap: Map[String, Either[SkeletonTracing, VolumeTracing]] = volumeTracingIds .zip(volumeTracings.map(versioned => Right[SkeletonTracing, VolumeTracing](versioned.value))) .toMap - } yield AnnotationWithTracings(annotation, skeletonTracingsMap ++ volumeTracingsMap, editableMappingsMap) + } yield AnnotationWithTracings(annotation, skeletonTracingsMap ++ volumeTracingsMap, Map.empty) } private def applyUpdates(annotation: AnnotationWithTracings, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala index 6ab093d315a..f61fc5bcf59 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala @@ -6,11 +6,8 @@ import com.scalableminds.webknossos.datastore.AgglomerateGraph.AgglomerateGraph import com.scalableminds.webknossos.datastore.ListOfLong.ListOfLong import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.controllers.Controller -import com.scalableminds.webknossos.datastore.services.{ - AccessTokenService, - EditableMappingSegmentListResult, - UserAccessRequest -} +import com.scalableminds.webknossos.datastore.services.{EditableMappingSegmentListResult, UserAccessRequest} +import com.scalableminds.webknossos.tracingstore.TracingStoreAccessTokenService import com.scalableminds.webknossos.tracingstore.annotation.{ AnnotationTransactionService, TSAnnotationService, @@ -30,7 +27,7 @@ import scala.concurrent.ExecutionContext class EditableMappingController @Inject()(volumeTracingService: VolumeTracingService, annotationService: TSAnnotationService, - accessTokenService: AccessTokenService, + accessTokenService: TracingStoreAccessTokenService, editableMappingService: EditableMappingService, annotationTransactionService: AnnotationTransactionService)( implicit ec: ExecutionContext, @@ -174,7 +171,11 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer tracing <- volumeTracingService.find(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - edges <- editableMappingService.agglomerateGraphMinCut(tracingId, request.body, remoteFallbackLayer) + editableMappingInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId) + edges <- editableMappingService.agglomerateGraphMinCut(tracingId, + editableMappingInfo, + request.body, + remoteFallbackLayer) } yield Ok(Json.toJson(edges)) } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index 3345d9d94be..19c7aeb5660 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -581,12 +581,14 @@ class EditableMappingService @Inject()( } } yield agglomerateGraph - def agglomerateGraphMinCut(tracingId: String, parameters: MinCutParameters, remoteFallbackLayer: RemoteFallbackLayer)( - implicit tc: TokenContext): Fox[List[EdgeWithPositions]] = + def agglomerateGraphMinCut( + tracingId: String, + editableMappingInfo: EditableMappingInfo, + parameters: MinCutParameters, + remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[List[EdgeWithPositions]] = for { // called here to ensure updates are applied - mapping <- getInfo(tracingId, version = None, remoteFallbackLayer) - agglomerateGraph <- getAgglomerateGraphForIdWithFallback(mapping, + agglomerateGraph <- getAgglomerateGraphForIdWithFallback(editableMappingInfo, tracingId, None, parameters.agglomerateId, diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 8787b4d32e9..5cb35653209 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -35,14 +35,11 @@ POST /volume/mergedFromIds @c POST /volume/mergedFromContents @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromContents(persist: Boolean) # Editable Mappings -# todo adapt frontend to mapping route prefix POST /mapping/:annotationId/:tracingId/makeMappingEditable @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.makeMappingEditable(annotationId: String, tracingId: String) GET /mapping/:annotationId/:tracingId/info @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.editableMappingInfo(annotationId: String, tracingId: String, version: Option[Long]) GET /mapping/:annotationId/:tracingId/segmentsForAgglomerate @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.segmentIdsForAgglomerate(annotationId: String, tracingId: String, agglomerateId: Long) POST /mapping/:annotationId/:tracingId/agglomeratesForSegments @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateIdsForSegments(annotationId: String, tracingId: String) -# todo adapt frontend to mapping route prefix POST /mapping/:annotationId/:tracingId/agglomerateGraphMinCut @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateGraphMinCut(annotationId: String, tracingId: String) -# todo adapt frontend to mapping route prefix POST /mapping/:annotationId/:tracingId/agglomerateGraphNeighbors @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateGraphNeighbors(annotationId: String, tracingId: String) # Zarr endpoints for volume annotations From 10206508ac3df4e1f1bb7d0651e11e15fd51cbf0 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 23 Sep 2024 10:27:55 +0200 Subject: [PATCH 070/150] =?UTF-8?q?add=20editable=20mapping=20to=20Annotat?= =?UTF-8?q?ionWithTracings=20when=20it=E2=80=99s=20created?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../annotation/AnnotationWithTracings.scala | 11 ++++- .../annotation/TSAnnotationService.scala | 49 +++++++++++++------ 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index bf5cc93b7fc..aa721910fba 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -12,7 +12,10 @@ import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ EditableMappingUpdater } import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.SkeletonUpdateAction -import com.scalableminds.webknossos.tracingstore.tracings.volume.ApplyableVolumeUpdateAction +import com.scalableminds.webknossos.tracingstore.tracings.volume.{ + ApplyableVolumeUpdateAction, + UpdateMappingNameVolumeAction +} import net.liftweb.common.{Box, Failure, Full} import scala.concurrent.ExecutionContext @@ -86,6 +89,12 @@ case class AnnotationWithTracings( this.copy(annotation = annotation.copy(version = newVersion), tracingsById = tracingsUpdated.toMap) } + def addEditableMapping(volumeTracingId: String, + editableMappingInfo: EditableMappingInfo, + updater: EditableMappingUpdater): AnnotationWithTracings = + this.copy(editableMappingsByTracingId = + editableMappingsByTracingId.updated(volumeTracingId, (editableMappingInfo, updater))) + def applySkeletonAction(a: SkeletonUpdateAction)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { skeletonTracing <- getSkeleton(a.actionTracingId) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 0cbe103f4d9..f2c14bb679b 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -29,7 +29,8 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ import com.scalableminds.webknossos.tracingstore.tracings.{ KeyValueStoreImplicits, RemoteFallbackLayer, - TracingDataStore + TracingDataStore, + VersionedKeyValuePair } import com.scalableminds.webknossos.tracingstore.{ TSRemoteDatastoreClient, @@ -81,8 +82,11 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl } yield updateActionGroups.reverse.flatten } - private def applyUpdate(annotationWithTracings: AnnotationWithTracings, updateAction: UpdateAction)( - implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = + private def applyUpdate( + annotationWithTracings: AnnotationWithTracings, + updateAction: UpdateAction, + targetVersion: Long // Note: this is not the target version of this one update, but of all pending + )(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = for { updated <- updateAction match { case a: AddLayerAnnotationUpdateAction => @@ -96,16 +100,16 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl case a: SkeletonUpdateAction => annotationWithTracings.applySkeletonAction(a) ?~> "applySkeletonAction.failed" case a: UpdateMappingNameVolumeAction if a.isEditable.contains(true) => - - TODO in case mapping is made editable, add it to the AnnotationWithTracings object here - + for { + withNewEditableMapping <- addEditableMapping(annotationWithTracings, a, targetVersion) + withApplyedVolumeAction <- withNewEditableMapping.applyVolumeAction(a) + } yield withApplyedVolumeAction case a: ApplyableVolumeUpdateAction => annotationWithTracings.applyVolumeAction(a) case a: EditableMappingUpdateAction => annotationWithTracings.applyEditableMappingAction(a) case _: BucketMutatingVolumeUpdateAction => Fox.successful(annotationWithTracings) // No-op, as bucket-mutating actions are performed eagerly, so not here. - // Note: UpdateBucketVolumeActions are not handled here, but instead eagerly on saving. case _ => Fox.failure(s"Received unsupported AnnotationUpdateAction action ${Json.toJson(updateAction)}") } } yield updated @@ -160,6 +164,18 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl tracing <- annotation.getEditableMappingInfo(tracingId) ?~> "getEditableMapping.failed" } yield tracing + // move the functions that construct the AnnotationWithTracigns elsewhere? + private def addEditableMapping(annotationWithTracings: AnnotationWithTracings, + action: UpdateMappingNameVolumeAction, + targetVersion: Long)(implicit tc: TokenContext): Fox[AnnotationWithTracings] = + for { + editableMappingInfo <- getEditableMappingInfoFromStore(action.actionTracingId, annotationWithTracings.version) + updater = editableMappingUpdaterFor(action.actionTracingId, + editableMappingInfo.value, + annotationWithTracings.version, + targetVersion) + } yield annotationWithTracings.addEditableMapping(action.actionTracingId, editableMappingInfo.value, updater) + private def applyPendingUpdates(annotation: AnnotationProto, annotationId: String, targetVersionOpt: Option[Long], @@ -191,8 +207,7 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl // TODO intersect with editable mapping updates? for { editableMappingInfos <- Fox.serialCombined(volumeIdsWithEditableMapping) { volumeTracingId => - tracingDataStore.editableMappingsInfo.get(volumeTracingId, version = Some(annotationWithTracings.version))( - fromProtoBytes[EditableMappingInfo]) + getEditableMappingInfoFromStore(volumeTracingId, annotationWithTracings.version) } } yield annotationWithTracings.copy( @@ -208,6 +223,11 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl .toMap) } + private def getEditableMappingInfoFromStore(volumeTracingId: String, + version: Long): Fox[VersionedKeyValuePair[EditableMappingInfo]] = + tracingDataStore.editableMappingsInfo.get(volumeTracingId, version = Some(version))( + fromProtoBytes[EditableMappingInfo]) + private def editableMappingUpdaterFor(tracingId: String, editableMappingInfo: EditableMappingInfo, currentMaterializedVersion: Long, @@ -262,10 +282,11 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl } yield AnnotationWithTracings(annotation, skeletonTracingsMap ++ volumeTracingsMap, Map.empty) } - private def applyUpdates(annotation: AnnotationWithTracings, - annotationId: String, - updates: List[UpdateAction], - targetVersion: Long)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = { + private def applyUpdates( + annotation: AnnotationWithTracings, + annotationId: String, + updates: List[UpdateAction], + targetVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = { def updateIter(annotationWithTracingsFox: Fox[AnnotationWithTracings], remainingUpdates: List[UpdateAction]): Fox[AnnotationWithTracings] = @@ -276,7 +297,7 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl case List() => Fox.successful(annotationWithTracings) case RevertToVersionUpdateAction(sourceVersion, _, _, _) :: tail => ??? - case update :: tail => updateIter(applyUpdate(annotationWithTracings, update), tail) + case update :: tail => updateIter(applyUpdate(annotationWithTracings, update, targetVersion), tail) } case _ => annotationWithTracingsFox } From e21bc6e8e85a05e6e2bd540809db9dfeb53c0285 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 23 Sep 2024 10:58:45 +0200 Subject: [PATCH 071/150] fix agglomeratesForSegments route --- frontend/javascripts/admin/admin_rest_api.ts | 3 ++- frontend/javascripts/oxalis/model/sagas/mapping_saga.ts | 1 + frontend/javascripts/oxalis/model/sagas/proofread_saga.ts | 6 ++++-- .../tracingstore/annotation/TSAnnotationService.scala | 2 +- .../controllers/EditableMappingController.scala | 7 ++----- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts index 75faa599ae8..9e3e4ebda6a 100644 --- a/frontend/javascripts/admin/admin_rest_api.ts +++ b/frontend/javascripts/admin/admin_rest_api.ts @@ -2122,6 +2122,7 @@ export async function getAgglomeratesForSegmentsFromDatastore( tracingStoreUrl: string, + annotationId: string, tracingId: string, segmentIds: Array, ): Promise { @@ -2131,7 +2132,7 @@ export async function getAgglomeratesForSegmentsFromTracingstore Request.receiveArraybuffer( - `${tracingStoreUrl}/tracings/mapping/${tracingId}/agglomeratesForSegments?token=${token}`, + `${tracingStoreUrl}/tracings/mapping/${annotationId}/${tracingId}/agglomeratesForSegments?token=${token}`, { method: "POST", body: segmentIdBuffer, diff --git a/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts b/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts index 1a0044c4370..ddfb14597ca 100644 --- a/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts @@ -455,6 +455,7 @@ function* updateLocalHdf5Mapping( ? yield* call( getAgglomeratesForSegmentsFromTracingstore, annotation.tracingStore.url, + annotation.annotationId, editableMapping.tracingId, Array.from(newSegmentIds), ) diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts index b9776143517..653f5f5919a 100644 --- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts @@ -1282,12 +1282,14 @@ function* splitAgglomerateInMapping( .filter(([_segmentId, agglomerateId]) => agglomerateId === comparableSourceAgglomerateId) .map(([segmentId, _agglomerateId]) => segmentId); - const tracingStoreHost = yield* select((state) => state.tracing.tracingStore.url); + const tracingStoreUrl = yield* select((state) => state.tracing.tracingStore.url); + const annotationId = yield* select((state) => state.tracing.annotationId); // Ask the server to map the (split) segment ids. This creates a partial mapping // that only contains these ids. const mappingAfterSplit = yield* call( getAgglomeratesForSegmentsFromTracingstore, - tracingStoreHost, + tracingStoreUrl, + annotationId, volumeTracingId, splitSegmentIds, ); diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index f2c14bb679b..c5e9fb6fb13 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -160,7 +160,7 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl implicit ec: ExecutionContext, tc: TokenContext): Fox[EditableMappingInfo] = for { - annotation <- getWithTracings(annotationId, version, List.empty, List(tracingId)) + annotation <- getWithTracings(annotationId, version, List.empty, List(tracingId)) ?~> "getWithTracings.failed" tracing <- annotation.getEditableMappingInfo(tracingId) ?~> "getEditableMapping.failed" } yield tracing diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala index f61fc5bcf59..77fe91f0be8 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala @@ -147,14 +147,11 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer tracing <- volumeTracingService.find(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - (editableMappingInfo, editableMappingVersion) <- editableMappingService.getInfoAndActualVersion( - tracingId, - requestedVersion = None, - remoteFallbackLayer = remoteFallbackLayer) + editableMappingInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId, version = None) relevantMapping: Map[Long, Long] <- editableMappingService.generateCombinedMappingForSegmentIds( request.body.items.toSet, editableMappingInfo, - editableMappingVersion, + tracing.version, tracingId, remoteFallbackLayer) agglomerateIdsSorted = relevantMapping.toSeq.sortBy(_._1).map(_._2) From f92d9330453666601d526b387c81305defbbbce4 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 23 Sep 2024 11:39:06 +0200 Subject: [PATCH 072/150] fix mincut route --- .../annotation/AnnotationWithTracings.scala | 2 +- .../EditableMappingController.scala | 21 ++++++++++++- .../controllers/VolumeTracingController.scala | 15 ---------- .../EditableMappingService.scala | 30 +++++++++---------- .../EditableMappingUpdater.scala | 14 ++++----- ...alableminds.webknossos.tracingstore.routes | 3 +- 6 files changed, 45 insertions(+), 40 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index aa721910fba..a20643fa51a 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -54,7 +54,7 @@ case class AnnotationWithTracings( (info, _) <- editableMappingsByTracingId.get(tracingId) } yield info - def getEditableMappingUpdater(tracingId: String): Option[EditableMappingUpdater] = + private def getEditableMappingUpdater(tracingId: String): Option[EditableMappingUpdater] = for { (_, updater) <- editableMappingsByTracingId.get(tracingId) } yield updater diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala index 77fe91f0be8..e93196c3b12 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala @@ -126,7 +126,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer _ <- editableMappingService.assertTracingHasEditableMapping(tracing) remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) agglomerateGraphBox: Box[AgglomerateGraph] <- editableMappingService - .getAgglomerateGraphForId(tracingId, agglomerateId, remoteFallbackLayer) + .getAgglomerateGraphForId(tracingId, tracing.version, agglomerateId, remoteFallbackLayer) .futureBox segmentIds <- agglomerateGraphBox match { case Full(agglomerateGraph) => Fox.successful(agglomerateGraph.segments) @@ -170,6 +170,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) editableMappingInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId) edges <- editableMappingService.agglomerateGraphMinCut(tracingId, + tracing.version, editableMappingInfo, request.body, remoteFallbackLayer) @@ -187,10 +188,28 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer _ <- editableMappingService.assertTracingHasEditableMapping(tracing) remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) (segmentId, edges) <- editableMappingService.agglomerateGraphNeighbors(tracingId, + tracing.version, request.body, remoteFallbackLayer) } yield Ok(Json.obj("segmentId" -> segmentId, "neighbors" -> Json.toJson(edges))) } } } + + def agglomerateSkeleton(annotationId: String, tracingId: String, agglomerateId: Long): Action[AnyContent] = + Action.async { implicit request => + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { + for { + tracing <- volumeTracingService.find(annotationId, tracingId) + _ <- bool2Fox(tracing.getHasEditableMapping) ?~> "Cannot query agglomerate skeleton for volume annotation" + editableMappingInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId) + remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) + agglomerateSkeletonBytes <- editableMappingService.getAgglomerateSkeletonWithFallback(tracingId, + tracing.version, + editableMappingInfo, + remoteFallbackLayer, + agglomerateId) + } yield Ok(agglomerateSkeletonBytes) + } + } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index d0cd69fd7cf..8a2c81a3006 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -328,21 +328,6 @@ class VolumeTracingController @Inject()( } } - def agglomerateSkeleton(annotationId: String, tracingId: String, agglomerateId: Long): Action[AnyContent] = - Action.async { implicit request => - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { - for { - tracing <- tracingService.find(annotationId, tracingId) - _ <- bool2Fox(tracing.getHasEditableMapping) ?~> "Cannot query agglomerate skeleton for volume annotation" - mappingName <- tracing.mappingName ?~> "annotation.agglomerateSkeleton.noMappingSet" - remoteFallbackLayer <- tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - agglomerateSkeletonBytes <- editableMappingService.getAgglomerateSkeletonWithFallback(mappingName, - remoteFallbackLayer, - agglomerateId) - } yield Ok(agglomerateSkeletonBytes) - } - } - def getSegmentVolume(annotationId: String, tracingId: String): Action[SegmentStatisticsParameters] = Action.async(validateJson[SegmentStatisticsParameters]) { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index 19c7aeb5660..0ff3d9d9922 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -416,12 +416,12 @@ class EditableMappingService @Inject()( } yield editableMappingForSegmentIds ++ baseMappingSubset def getAgglomerateSkeletonWithFallback(tracingId: String, + version: Long, + editableMappingInfo: EditableMappingInfo, remoteFallbackLayer: RemoteFallbackLayer, agglomerateId: Long)(implicit tc: TokenContext): Fox[Array[Byte]] = for { - // called here to ensure updates are applied - editableMappingInfo <- getInfo(tracingId, version = None, remoteFallbackLayer) - agglomerateGraphBox <- getAgglomerateGraphForId(tracingId, agglomerateId, remoteFallbackLayer).futureBox + agglomerateGraphBox <- getAgglomerateGraphForId(tracingId, version, agglomerateId, remoteFallbackLayer).futureBox skeletonBytes <- agglomerateGraphBox match { case Full(agglomerateGraph) => Fox.successful(agglomerateGraphToSkeleton(tracingId, agglomerateGraph, remoteFallbackLayer, agglomerateId)) @@ -546,19 +546,17 @@ class EditableMappingService @Inject()( } yield result def getAgglomerateGraphForId( - mappingId: String, + tracingId: String, + version: Long, agglomerateId: Long, - remoteFallbackLayer: RemoteFallbackLayer, - requestedVersion: Option[Long] = None)(implicit tc: TokenContext): Fox[AgglomerateGraph] = + remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[AgglomerateGraph] = for { - // called here to ensure updates are applied - (_, version) <- getInfoAndActualVersion(mappingId, requestedVersion, remoteFallbackLayer) agglomerateGraph <- agglomerateToGraphCache.getOrLoad( - (mappingId, agglomerateId, version), + (tracingId, agglomerateId, version), _ => for { graphBytes: VersionedKeyValuePair[Array[Byte]] <- tracingDataStore.editableMappingsAgglomerateToGraph - .get(agglomerateGraphKey(mappingId, agglomerateId), Some(version), mayBeEmpty = Some(true)) + .get(agglomerateGraphKey(tracingId, agglomerateId), Some(version), mayBeEmpty = Some(true)) graphParsed <- if (isRevertedElement(graphBytes.value)) Fox.empty else fromProtoBytes[AgglomerateGraph](graphBytes.value).toFox } yield graphParsed @@ -568,11 +566,11 @@ class EditableMappingService @Inject()( def getAgglomerateGraphForIdWithFallback( mapping: EditableMappingInfo, tracingId: String, - version: Option[Long], + version: Long, agglomerateId: Long, remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[AgglomerateGraph] = for { - agglomerateGraphBox <- getAgglomerateGraphForId(tracingId, agglomerateId, remoteFallbackLayer, version).futureBox + agglomerateGraphBox <- getAgglomerateGraphForId(tracingId, version, agglomerateId, remoteFallbackLayer).futureBox agglomerateGraph <- agglomerateGraphBox match { case Full(agglomerateGraph) => Fox.successful(agglomerateGraph) case Empty => @@ -583,6 +581,7 @@ class EditableMappingService @Inject()( def agglomerateGraphMinCut( tracingId: String, + version: Long, editableMappingInfo: EditableMappingInfo, parameters: MinCutParameters, remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[List[EdgeWithPositions]] = @@ -590,9 +589,9 @@ class EditableMappingService @Inject()( // called here to ensure updates are applied agglomerateGraph <- getAgglomerateGraphForIdWithFallback(editableMappingInfo, tracingId, - None, + version, parameters.agglomerateId, - remoteFallbackLayer) + remoteFallbackLayer) ?~> "getAgglomerateGraph.failed" edgesToCut <- minCut(agglomerateGraph, parameters.segmentId1, parameters.segmentId2) ?~> "Could not calculate min-cut on agglomerate graph." edgesWithPositions = annotateEdgesWithPositions(edgesToCut, agglomerateGraph) } yield edgesWithPositions @@ -651,6 +650,7 @@ class EditableMappingService @Inject()( def agglomerateGraphNeighbors( tracingId: String, + version: Long, parameters: NeighborsParameters, remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[(Long, Seq[NodeWithPosition])] = for { @@ -658,7 +658,7 @@ class EditableMappingService @Inject()( mapping <- getInfo(tracingId, version = None, remoteFallbackLayer) agglomerateGraph <- getAgglomerateGraphForIdWithFallback(mapping, tracingId, - None, + version, parameters.agglomerateId, remoteFallbackLayer) neighborNodes = neighbors(agglomerateGraph, parameters.segmentId) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index 8334d5138e1..3d5a2d0b5d3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -235,9 +235,11 @@ class EditableMappingUpdater( val key = agglomerateGraphKey(tracingId, agglomerateId) val fromBufferOpt = getFromAgglomerateToGraphBuffer(key) fromBufferOpt.map(Fox.successful(_)).getOrElse { - editableMappingService - .getAgglomerateGraphForIdWithFallback(mapping, tracingId, Some(oldVersion), agglomerateId, remoteFallbackLayer, - )(tokenContext) + editableMappingService.getAgglomerateGraphForIdWithFallback(mapping, + tracingId, + oldVersion, + agglomerateId, + remoteFallbackLayer)(tokenContext) } } @@ -441,10 +443,8 @@ class EditableMappingUpdater( for { agglomerateId <- agglomerateIdFromAgglomerateGraphKey(graphKey) _ <- editableMappingService - .getAgglomerateGraphForId(tracingId, - agglomerateId, - remoteFallbackLayer, - Some(revertAction.sourceVersion))(tokenContext) + .getAgglomerateGraphForId(tracingId, revertAction.sourceVersion, agglomerateId, remoteFallbackLayer)( + tokenContext) .futureBox .map { case Full(graphData) => agglomerateToGraphBuffer.put(graphKey, (graphData, false)) diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 5cb35653209..a7b9619e365 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -27,7 +27,6 @@ POST /volume/:annotationId/:tracingId/segmentIndex/:segmentId @c POST /volume/:annotationId/:tracingId/importVolumeData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.importVolumeData(annotationId: String, tracingId: String) POST /volume/:annotationId/:tracingId/addSegmentIndex @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.addSegmentIndex(annotationId: String, tracingId: String, dryRun: Boolean) GET /volume/:annotationId/:tracingId/findData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.findData(annotationId: String, tracingId: String) -GET /volume/:annotationId/:tracingId/agglomerateSkeleton/:agglomerateId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.agglomerateSkeleton(annotationId: String, tracingId: String, agglomerateId: Long) POST /volume/:annotationId/:tracingId/segmentStatistics/volume @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentVolume(annotationId: String, tracingId: String) POST /volume/:annotationId/:tracingId/segmentStatistics/boundingBox @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentBoundingBox(annotationId: String, tracingId: String) POST /volume/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getMultiple @@ -41,6 +40,8 @@ GET /mapping/:annotationId/:tracingId/segmentsForAgglomerate @c POST /mapping/:annotationId/:tracingId/agglomeratesForSegments @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateIdsForSegments(annotationId: String, tracingId: String) POST /mapping/:annotationId/:tracingId/agglomerateGraphMinCut @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateGraphMinCut(annotationId: String, tracingId: String) POST /mapping/:annotationId/:tracingId/agglomerateGraphNeighbors @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateGraphNeighbors(annotationId: String, tracingId: String) +# TODO rename +GET /volume/:annotationId/:tracingId/agglomerateSkeleton/:agglomerateId @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateSkeleton(annotationId: String, tracingId: String, agglomerateId: Long) # Zarr endpoints for volume annotations # Zarr version 2 From 19f39e5b96b6386572b6feb3681e80522190528d Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 23 Sep 2024 11:49:07 +0200 Subject: [PATCH 073/150] wip build correct remoteFallbackLayer for updater --- .../annotation/TSAnnotationService.scala | 61 +++++++++++-------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index c5e9fb6fb13..ef59199f212 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -27,6 +27,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ VolumeUpdateAction } import com.scalableminds.webknossos.tracingstore.tracings.{ + FallbackDataHelper, KeyValueStoreImplicits, RemoteFallbackLayer, TracingDataStore, @@ -49,6 +50,7 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl remoteDatastoreClient: TSRemoteDatastoreClient, tracingDataStore: TracingDataStore) extends KeyValueStoreImplicits + with FallbackDataHelper with LazyLogging { def reportUpdates(annotationId: String, updateGroups: List[UpdateActionGroup])(implicit tc: TokenContext): Fox[Unit] = @@ -165,15 +167,18 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl } yield tracing // move the functions that construct the AnnotationWithTracigns elsewhere? - private def addEditableMapping(annotationWithTracings: AnnotationWithTracings, - action: UpdateMappingNameVolumeAction, - targetVersion: Long)(implicit tc: TokenContext): Fox[AnnotationWithTracings] = + private def addEditableMapping( + annotationWithTracings: AnnotationWithTracings, + action: UpdateMappingNameVolumeAction, + targetVersion: Long)(implicit tc: TokenContext, ec: ExecutionContext): Fox[AnnotationWithTracings] = for { editableMappingInfo <- getEditableMappingInfoFromStore(action.actionTracingId, annotationWithTracings.version) - updater = editableMappingUpdaterFor(action.actionTracingId, - editableMappingInfo.value, - annotationWithTracings.version, - targetVersion) + volumeTracing <- annotationWithTracings.getVolume(action.actionTracingId).toFox + updater <- editableMappingUpdaterFor(action.actionTracingId, + volumeTracing, + editableMappingInfo.value, + annotationWithTracings.version, + targetVersion) } yield annotationWithTracings.addEditableMapping(action.actionTracingId, editableMappingInfo.value, updater) private def applyPendingUpdates(annotation: AnnotationProto, @@ -216,7 +221,9 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl keyValuePair => (keyValuePair.key, (keyValuePair.value, + // TODO this returns Fox now editableMappingUpdaterFor(keyValuePair.key, + // TODO fetch volume tracing keyValuePair.value, currentMaterializedVersion, targetVersion)))) @@ -228,25 +235,27 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl tracingDataStore.editableMappingsInfo.get(volumeTracingId, version = Some(version))( fromProtoBytes[EditableMappingInfo]) - private def editableMappingUpdaterFor(tracingId: String, - editableMappingInfo: EditableMappingInfo, - currentMaterializedVersion: Long, - targetVersion: Long)(implicit tc: TokenContext): EditableMappingUpdater = { - val remoteFallbackLayer - : RemoteFallbackLayer = RemoteFallbackLayer("todo", "todo", "todo", ElementClassProto.uint8) // TODO - new EditableMappingUpdater( - tracingId, - editableMappingInfo.baseMappingName, - currentMaterializedVersion, - targetVersion, - remoteFallbackLayer, - tc, - remoteDatastoreClient, - editableMappingService, - tracingDataStore, - relyOnAgglomerateIds = false // TODO - ) - } + private def editableMappingUpdaterFor( + tracingId: String, + volumeTracing: VolumeTracing, + editableMappingInfo: EditableMappingInfo, + currentMaterializedVersion: Long, + targetVersion: Long)(implicit tc: TokenContext, ec: ExecutionContext): Fox[EditableMappingUpdater] = + for { + remoteFallbackLayer <- remoteFallbackLayerFromVolumeTracing(volumeTracing, tracingId) + } yield + new EditableMappingUpdater( + tracingId, + editableMappingInfo.baseMappingName, + currentMaterializedVersion, + targetVersion, + remoteFallbackLayer, + tc, + remoteDatastoreClient, + editableMappingService, + tracingDataStore, + relyOnAgglomerateIds = false // TODO should we? + ) private def findTracingsForUpdates( annotation: AnnotationProto, From 831bf7a3f710c4eda37df75d7978e30cab20d295 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 23 Sep 2024 13:01:12 +0200 Subject: [PATCH 074/150] pass volumetracing info to updater --- .../annotation/AnnotationWithTracings.scala | 4 +-- .../annotation/TSAnnotationService.scala | 35 ++++++++----------- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index a20643fa51a..d0435b1b35e 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -43,9 +43,9 @@ case class AnnotationWithTracings( } } yield volumeTracing - def volumesIdsThatHaveEditableMapping: List[String] = + def volumesThatHaveEditableMapping: List[(VolumeTracing, String)] = tracingsById.view.flatMap { - case (id, Right(vt: VolumeTracing)) if vt.getHasEditableMapping => Some(id) + case (id, Right(vt: VolumeTracing)) if vt.getHasEditableMapping => Some((vt, id)) case _ => None }.toList diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index ef59199f212..e1b3ae531dd 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -45,9 +45,9 @@ import play.api.libs.json.{JsObject, JsValue, Json} import javax.inject.Inject import scala.concurrent.ExecutionContext -class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosClient, +class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknossosClient, editableMappingService: EditableMappingService, - remoteDatastoreClient: TSRemoteDatastoreClient, + val remoteDatastoreClient: TSRemoteDatastoreClient, tracingDataStore: TracingDataStore) extends KeyValueStoreImplicits with FallbackDataHelper @@ -207,27 +207,22 @@ class TSAnnotationService @Inject()(remoteWebknossosClient: TSRemoteWebknossosCl updates: List[UpdateAction], currentMaterializedVersion: Long, targetVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext) = { - val volumeIdsWithEditableMapping = annotationWithTracings.volumesIdsThatHaveEditableMapping - logger.info(s"fetching editable mappings ${volumeIdsWithEditableMapping.mkString(",")}") + val volumeWithEditableMapping = annotationWithTracings.volumesThatHaveEditableMapping + logger.info(s"fetching editable mappings ${volumeWithEditableMapping.map(_._2).mkString(",")}") // TODO intersect with editable mapping updates? for { - editableMappingInfos <- Fox.serialCombined(volumeIdsWithEditableMapping) { volumeTracingId => - getEditableMappingInfoFromStore(volumeTracingId, annotationWithTracings.version) + idInfoUpdaterTuples <- Fox.serialCombined(volumeWithEditableMapping) { + case (volumeTracing, volumeTracingId) => + for { + editableMappingInfo <- getEditableMappingInfoFromStore(volumeTracingId, annotationWithTracings.version) + updater <- editableMappingUpdaterFor(volumeTracingId, + volumeTracing, + editableMappingInfo.value, + currentMaterializedVersion, + targetVersion) + } yield (editableMappingInfo.key, (editableMappingInfo.value, updater)) } - } yield - annotationWithTracings.copy( - editableMappingsByTracingId = editableMappingInfos - .map( - keyValuePair => - (keyValuePair.key, - (keyValuePair.value, - // TODO this returns Fox now - editableMappingUpdaterFor(keyValuePair.key, - // TODO fetch volume tracing - keyValuePair.value, - currentMaterializedVersion, - targetVersion)))) - .toMap) + } yield annotationWithTracings.copy(editableMappingsByTracingId = idInfoUpdaterTuples.toMap) } private def getEditableMappingInfoFromStore(volumeTracingId: String, From e5cd917c3ccf9a704feb45ae1cf471be7ae7b603 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 23 Sep 2024 14:47:07 +0200 Subject: [PATCH 075/150] flush editable mapping updater --- .../services/DSRemoteTracingstoreClient.scala | 2 ++ .../annotation/AnnotationWithTracings.scala | 8 ++++++++ .../tracingstore/annotation/TSAnnotationService.scala | 3 ++- .../editablemapping/EditableMappingUpdater.scala | 10 ++++++++-- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala index 5bd69d4d7c9..42473d29a21 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala @@ -70,9 +70,11 @@ class DSRemoteTracingstoreClient @Inject()( rpc(s"$tracingStoreUri/tracings/volume/${getZarrVersionDependantSubPath(zarrVersion)}/json/$tracingId").withTokenFromContext .getWithJsonResponse[List[String]] + // TODO annotation id def getZGroup(tracingId: String, tracingStoreUri: String)(implicit tc: TokenContext): Fox[JsObject] = rpc(s"$tracingStoreUri/tracings/volume/zarr/$tracingId/.zgroup").withTokenFromContext.getWithJsonResponse[JsObject] + // TODO annotation id def getEditableMappingSegmentIdsForAgglomerate(tracingStoreUri: String, tracingId: String, agglomerateId: Long)( implicit tc: TokenContext): Fox[EditableMappingSegmentListResult] = rpc(s"$tracingStoreUri/tracings/mapping/$tracingId/segmentsForAgglomerate") diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index d0435b1b35e..01022928517 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -119,4 +119,12 @@ case class AnnotationWithTracings( } yield this.copy( editableMappingsByTracingId = editableMappingsByTracingId.updated(a.actionTracingId, (updated, updater))) + + def flushBufferedUpdates()(implicit ec: ExecutionContext): Fox[Unit] = { + val updaters = editableMappingsByTracingId.values.map(_._2).toList + for { + _ <- Fox.serialCombined(updaters)(updater => updater.flushBuffersToFossil()) + } yield () + } + } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index e1b3ae531dd..09842a764ba 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -310,7 +310,8 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss else { for { updated <- updateIter(Some(annotation), updates) - // TODO flush editable mapping updaters + _ <- updated.flushBufferedUpdates() + // todo: save materialized tracings + editable mapping info } yield updated.withVersion(targetVersion) } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index 3d5a2d0b5d3..e40475db5e5 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -60,13 +60,19 @@ class EditableMappingUpdater( dry: Boolean = false)(implicit ec: ExecutionContext): Fox[EditableMappingInfo] = for { updatedEditableMappingInfo: EditableMappingInfo <- updateIter(Some(existingEditabeMappingInfo), updates) - _ <- Fox.runIf(!dry)(flushToFossil(updatedEditableMappingInfo)) + _ <- Fox.runIf(!dry)(flushBuffersToFossil()) + _ <- Fox.runIf(!dry)(flushUpdatedInfoToFossil(updatedEditableMappingInfo)) } yield updatedEditableMappingInfo - private def flushToFossil(updatedEditableMappingInfo: EditableMappingInfo)(implicit ec: ExecutionContext): Fox[Unit] = + def flushBuffersToFossil()(implicit ec: ExecutionContext): Fox[Unit] = for { _ <- Fox.serialCombined(segmentToAgglomerateBuffer.keys.toList)(flushSegmentToAgglomerateChunk) _ <- Fox.serialCombined(agglomerateToGraphBuffer.keys.toList)(flushAgglomerateGraph) + } yield () + + private def flushUpdatedInfoToFossil(updatedEditableMappingInfo: EditableMappingInfo)( + implicit ec: ExecutionContext): Fox[Unit] = + for { _ <- tracingDataStore.editableMappingsInfo.put(tracingId, newVersion, updatedEditableMappingInfo) } yield () From e4825a604cbb455d512de33dd0d5c39cbba399bc Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 23 Sep 2024 14:58:47 +0200 Subject: [PATCH 076/150] imports --- .../tracingstore/annotation/AnnotationWithTracings.scala | 5 +---- .../tracingstore/annotation/TSAnnotationService.scala | 2 -- .../editablemapping/EditableMappingService.scala | 9 ++++----- .../editablemapping/EditableMappingUpdater.scala | 3 +-- .../tracings/volume/VolumeUpdateActions.scala | 1 - 5 files changed, 6 insertions(+), 14 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index 01022928517..e8c59a7010d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -12,10 +12,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ EditableMappingUpdater } import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.SkeletonUpdateAction -import com.scalableminds.webknossos.tracingstore.tracings.volume.{ - ApplyableVolumeUpdateAction, - UpdateMappingNameVolumeAction -} +import com.scalableminds.webknossos.tracingstore.tracings.volume.ApplyableVolumeUpdateAction import net.liftweb.common.{Box, Failure, Full} import scala.concurrent.ExecutionContext diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 09842a764ba..984cfb6cef9 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -8,7 +8,6 @@ import com.scalableminds.webknossos.datastore.Annotation.AnnotationProto import com.scalableminds.webknossos.datastore.EditableMappingInfo.EditableMappingInfo import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing -import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing.ElementClassProto import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ EditableMappingService, EditableMappingUpdateAction, @@ -29,7 +28,6 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ import com.scalableminds.webknossos.tracingstore.tracings.{ FallbackDataHelper, KeyValueStoreImplicits, - RemoteFallbackLayer, TracingDataStore, VersionedKeyValuePair } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index 0ff3d9d9922..86dd974e376 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -545,11 +545,10 @@ class EditableMappingService @Inject()( result <- adHocMeshService.requestAdHocMeshViaActor(adHocMeshRequest) } yield result - def getAgglomerateGraphForId( - tracingId: String, - version: Long, - agglomerateId: Long, - remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[AgglomerateGraph] = + def getAgglomerateGraphForId(tracingId: String, + version: Long, + agglomerateId: Long, + remoteFallbackLayer: RemoteFallbackLayer): Fox[AgglomerateGraph] = for { agglomerateGraph <- agglomerateToGraphCache.getOrLoad( (tracingId, agglomerateId, version), diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index e40475db5e5..97ee1af37d8 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -70,8 +70,7 @@ class EditableMappingUpdater( _ <- Fox.serialCombined(agglomerateToGraphBuffer.keys.toList)(flushAgglomerateGraph) } yield () - private def flushUpdatedInfoToFossil(updatedEditableMappingInfo: EditableMappingInfo)( - implicit ec: ExecutionContext): Fox[Unit] = + private def flushUpdatedInfoToFossil(updatedEditableMappingInfo: EditableMappingInfo): Fox[Unit] = for { _ <- tracingDataStore.editableMappingsInfo.put(tracingId, newVersion, updatedEditableMappingInfo) } yield () diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala index d00a7707d10..7cfc4b44a09 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala @@ -1,6 +1,5 @@ package com.scalableminds.webknossos.tracingstore.tracings.volume -import java.util.Base64 import com.scalableminds.util.geometry.{Vec3Double, Vec3Int} import com.scalableminds.webknossos.datastore.VolumeTracing.{Segment, SegmentGroup, VolumeTracing} import com.scalableminds.webknossos.datastore.geometry.NamedBoundingBoxProto From 4cc4647ed636e38b338dec70c908de8bbeca453d Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 24 Sep 2024 13:51:07 +0200 Subject: [PATCH 077/150] fix segment stats request uri --- .../right-border-tabs/segments_tab/segments_view_helper.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/javascripts/oxalis/view/right-border-tabs/segments_tab/segments_view_helper.tsx b/frontend/javascripts/oxalis/view/right-border-tabs/segments_tab/segments_view_helper.tsx index 0440554ef95..818b990e681 100644 --- a/frontend/javascripts/oxalis/view/right-border-tabs/segments_tab/segments_view_helper.tsx +++ b/frontend/javascripts/oxalis/view/right-border-tabs/segments_tab/segments_view_helper.tsx @@ -50,7 +50,8 @@ export function getVolumeRequestUrl( return `${dataset.dataStore.url}/data/datasets/${dataset.owningOrganization}/${dataset.name}/layers/${visibleSegmentationLayer.name}`; } else { const tracingStoreHost = tracing?.tracingStore.url; - return `${tracingStoreHost}/tracings/volume/${tracingId}`; + const annotationId = tracing?.annotationId; + return `${tracingStoreHost}/tracings/volume/${annotationId}/${tracingId}`; } } From 8c11da294dccf98e84c26fec1918a8064f17faa8 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 24 Sep 2024 14:23:57 +0200 Subject: [PATCH 078/150] wip: fix some more functions --- .../annotation/TSAnnotationService.scala | 21 ++- .../EditableMappingController.scala | 2 + .../controllers/VolumeTracingController.scala | 5 +- .../EditableMappingService.scala | 138 ++---------------- .../EditableMappingUpdater.scala | 11 +- .../volume/VolumeTracingService.scala | 2 +- 6 files changed, 41 insertions(+), 138 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 984cfb6cef9..0e77b6177e7 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -82,7 +82,9 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } yield updateActionGroups.reverse.flatten } + // TODO option to dry apply? private def applyUpdate( + annotationId: String, annotationWithTracings: AnnotationWithTracings, updateAction: UpdateAction, targetVersion: Long // Note: this is not the target version of this one update, but of all pending @@ -101,7 +103,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss annotationWithTracings.applySkeletonAction(a) ?~> "applySkeletonAction.failed" case a: UpdateMappingNameVolumeAction if a.isEditable.contains(true) => for { - withNewEditableMapping <- addEditableMapping(annotationWithTracings, a, targetVersion) + withNewEditableMapping <- addEditableMapping(annotationId, annotationWithTracings, a, targetVersion) withApplyedVolumeAction <- withNewEditableMapping.applyVolumeAction(a) } yield withApplyedVolumeAction case a: ApplyableVolumeUpdateAction => @@ -166,13 +168,15 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss // move the functions that construct the AnnotationWithTracigns elsewhere? private def addEditableMapping( + annotationId: String, annotationWithTracings: AnnotationWithTracings, action: UpdateMappingNameVolumeAction, targetVersion: Long)(implicit tc: TokenContext, ec: ExecutionContext): Fox[AnnotationWithTracings] = for { editableMappingInfo <- getEditableMappingInfoFromStore(action.actionTracingId, annotationWithTracings.version) volumeTracing <- annotationWithTracings.getVolume(action.actionTracingId).toFox - updater <- editableMappingUpdaterFor(action.actionTracingId, + updater <- editableMappingUpdaterFor(annotationId, + action.actionTracingId, volumeTracing, editableMappingInfo.value, annotationWithTracings.version, @@ -193,7 +197,8 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss updates, requestedSkeletonTracingIds, requestedVolumeTracingIds) ?~> "findTracingsForUpdates.failed" - annotationWithTracingsAndMappings <- findEditableMappingsForUpdates(annotationWithTracings, + annotationWithTracingsAndMappings <- findEditableMappingsForUpdates(annotationId, + annotationWithTracings, updates, annotation.version, targetVersion) @@ -201,6 +206,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } yield updated private def findEditableMappingsForUpdates( // TODO integrate with findTracings? + annotationId: String, annotationWithTracings: AnnotationWithTracings, updates: List[UpdateAction], currentMaterializedVersion: Long, @@ -213,7 +219,8 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss case (volumeTracing, volumeTracingId) => for { editableMappingInfo <- getEditableMappingInfoFromStore(volumeTracingId, annotationWithTracings.version) - updater <- editableMappingUpdaterFor(volumeTracingId, + updater <- editableMappingUpdaterFor(annotationId, + volumeTracingId, volumeTracing, editableMappingInfo.value, currentMaterializedVersion, @@ -229,6 +236,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss fromProtoBytes[EditableMappingInfo]) private def editableMappingUpdaterFor( + annotationId: String, tracingId: String, volumeTracing: VolumeTracing, editableMappingInfo: EditableMappingInfo, @@ -238,6 +246,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss remoteFallbackLayer <- remoteFallbackLayerFromVolumeTracing(volumeTracing, tracingId) } yield new EditableMappingUpdater( + annotationId, tracingId, editableMappingInfo.baseMappingName, currentMaterializedVersion, @@ -246,6 +255,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss tc, remoteDatastoreClient, editableMappingService, + this, tracingDataStore, relyOnAgglomerateIds = false // TODO should we? ) @@ -299,7 +309,8 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss case List() => Fox.successful(annotationWithTracings) case RevertToVersionUpdateAction(sourceVersion, _, _, _) :: tail => ??? - case update :: tail => updateIter(applyUpdate(annotationWithTracings, update, targetVersion), tail) + case update :: tail => + updateIter(applyUpdate(annotationId, annotationWithTracings, update, targetVersion), tail) } case _ => annotationWithTracingsFox } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala index e93196c3b12..24e41a83f88 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala @@ -187,7 +187,9 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer tracing <- volumeTracingService.find(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) + editableMappingInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId) (segmentId, edges) <- editableMappingService.agglomerateGraphNeighbors(tracingId, + editableMappingInfo, tracing.version, request.body, remoteFallbackLayer) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 8a2c81a3006..b8233ba03cd 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -207,8 +207,9 @@ class VolumeTracingController @Inject()( remoteFallbackLayerOpt <- Fox.runIf(tracing.getHasEditableMapping)( tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId)) newTracingId = tracingService.generateTracingId - _ <- Fox.runIf(tracing.getHasEditableMapping)( - editableMappingService.duplicate(tracingId, newTracingId, version = None, remoteFallbackLayerOpt)) + // TODO + /*_ <- Fox.runIf(tracing.getHasEditableMapping)( + editableMappingService.duplicate(tracingId, newTracingId, version = None, remoteFallbackLayerOpt))*/ (newId, newTracing) <- tracingService.duplicate( annotationId, tracingId, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index 86dd974e376..c48834d0992 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -108,6 +108,7 @@ class EditableMappingService @Inject()( adHocMeshServiceHolder.tracingStoreAdHocMeshConfig = (binaryDataService, 30 seconds, 1) private val adHocMeshService: AdHocMeshService = adHocMeshServiceHolder.tracingStoreAdHocMeshService + // TODO private lazy val materializedInfoCache: AlfuCache[(String, Long), EditableMappingInfo] = AlfuCache(maxCapacity = 100) private lazy val segmentToAgglomerateChunkCache: AlfuCache[(String, Long, Long), Seq[(Long, Long)]] = @@ -135,6 +136,7 @@ class EditableMappingService @Inject()( } yield newEditableMappingInfo } + /* TODO def duplicate(sourceTracingId: String, newTracingId: String, version: Option[Long], @@ -153,6 +155,8 @@ class EditableMappingService @Inject()( } } yield () + */ + private def duplicateSegmentToAgglomerate(sourceTracingId: String, newId: String, newVersion: Long): Fox[Unit] = { val iterator = new VersionedFossilDbIterator(sourceTracingId, @@ -187,135 +191,11 @@ class EditableMappingService @Inject()( } yield () } - def getInfo(tracingId: String, version: Option[Long] = None, remoteFallbackLayer: RemoteFallbackLayer)( - implicit tc: TokenContext): Fox[EditableMappingInfo] = - for { - (info, _) <- getInfoAndActualVersion(tracingId, version, remoteFallbackLayer) - } yield info - def assertTracingHasEditableMapping(tracing: VolumeTracing)(implicit ec: ExecutionContext): Fox[Unit] = bool2Fox(tracing.getHasEditableMapping) ?~> "annotation.volume.noEditableMapping" def getBaseMappingName(tracingId: String): Fox[Option[String]] = - for { - desiredVersion <- getClosestMaterializableVersionOrZero(tracingId, None) - infoBox <- getClosestMaterialized(tracingId, desiredVersion).futureBox - } yield - infoBox match { - case Full(info) => Some(info.value.baseMappingName) - case _ => None - } - - def getInfoAndActualVersion( - tracingId: String, - requestedVersion: Option[Long] = None, - remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[(EditableMappingInfo, Long)] = - for { - desiredVersion <- getClosestMaterializableVersionOrZero(tracingId, requestedVersion) - materializedInfo <- materializedInfoCache.getOrLoad( - (tracingId, desiredVersion), - _ => applyPendingUpdates(tracingId, desiredVersion, remoteFallbackLayer)) - } yield (materializedInfo, desiredVersion) - - def update(tracingId: String, - updateActionGroup: UpdateActionGroup, - newVersion: Long, - remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[Unit] = - for { - actionsWithTimestamp <- Fox.successful(updateActionGroup.actions.map(_.addTimestamp(updateActionGroup.timestamp))) - _ <- dryApplyUpdates(tracingId, newVersion, actionsWithTimestamp, remoteFallbackLayer) ?~> "editableMapping.dryUpdate.failed" - _ <- tracingDataStore.editableMappingUpdates.put(tracingId, newVersion, actionsWithTimestamp) - } yield () - - private def dryApplyUpdates(tracingId: String, - newVersion: Long, - updates: List[UpdateAction], - remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[Unit] = - for { - (previousInfo, previousVersion) <- getInfoAndActualVersion(tracingId, None, remoteFallbackLayer) - updater = new EditableMappingUpdater( - tracingId, - previousInfo.baseMappingName, - previousVersion, - newVersion, - remoteFallbackLayer, - tc, - remoteDatastoreClient, - this, - tracingDataStore, - relyOnAgglomerateIds = updates.length <= 1 - ) - _ <- updater.applyUpdatesAndSave(previousInfo, updates, dry = true) ?~> "editableMapping.update.failed" - } yield () - - def applyPendingUpdates(tracingId: String, desiredVersion: Long, remoteFallbackLayer: RemoteFallbackLayer)( - implicit tc: TokenContext): Fox[EditableMappingInfo] = - for { - closestMaterializedWithVersion <- getClosestMaterialized(tracingId, desiredVersion) - updatedEditableMappingInfo: EditableMappingInfo <- if (desiredVersion == closestMaterializedWithVersion.version) - Fox.successful(closestMaterializedWithVersion.value) - else - for { - pendingUpdates <- getPendingUpdates(tracingId, closestMaterializedWithVersion.version, desiredVersion) - updater = new EditableMappingUpdater( - tracingId, - closestMaterializedWithVersion.value.baseMappingName, - closestMaterializedWithVersion.version, - desiredVersion, - remoteFallbackLayer, - tc, - remoteDatastoreClient, - this, - tracingDataStore, - relyOnAgglomerateIds = pendingUpdates.length <= 1 - ) - updated <- updater.applyUpdatesAndSave(closestMaterializedWithVersion.value, pendingUpdates) - } yield updated - } yield updatedEditableMappingInfo - - private def getClosestMaterialized(tracingId: String, - desiredVersion: Long): Fox[VersionedKeyValuePair[EditableMappingInfo]] = - tracingDataStore.editableMappingsInfo.get(tracingId, version = Some(desiredVersion))( - fromProtoBytes[EditableMappingInfo]) - - def getClosestMaterializableVersionOrZero(tracingId: String, desiredVersion: Option[Long]): Fox[Long] = - tracingDataStore.editableMappingUpdates.getVersion(tracingId, - version = desiredVersion, - mayBeEmpty = Some(true), - emptyFallback = Some(0L)) - - private def getPendingUpdates(tracingId: String, - closestMaterializedVersion: Long, - closestMaterializableVersion: Long): Fox[List[UpdateAction]] = - if (closestMaterializableVersion == closestMaterializedVersion) { - Fox.successful(List.empty) - } else { - for { - updates <- getUpdateActionsWithVersions(tracingId, - newestVersion = closestMaterializableVersion, - oldestVersion = closestMaterializedVersion + 1L) - } yield updates.map(_._2).reverse.flatten - } - - private def getUpdateActionsWithVersions(tracingId: String, - newestVersion: Long, - oldestVersion: Long): Fox[List[(Long, List[UpdateAction])]] = { - val batchRanges = batchRangeInclusive(oldestVersion, newestVersion, batchSize = 100) - for { - updateActionBatches <- Fox.serialCombined(batchRanges.toList) { batchRange => - val batchFrom = batchRange._1 - val batchTo = batchRange._2 - for { - res <- tracingDataStore.editableMappingUpdates.getMultipleVersionsAsVersionValueTuple[List[UpdateAction]]( - tracingId, - Some(batchTo), - Some(batchFrom) - )(fromJsonBytes[List[UpdateAction]]) - } yield res - } ?~> "Failed to fetch editable mapping update actions from fossilDB" - flat = updateActionBatches.flatten - } yield flat - } + Fox.successful(None) def findSegmentIdAtPositionIfNeeded(remoteFallbackLayer: RemoteFallbackLayer, positionOpt: Option[Vec3Int], @@ -649,13 +529,12 @@ class EditableMappingService @Inject()( def agglomerateGraphNeighbors( tracingId: String, + editableMappingInfo: EditableMappingInfo, version: Long, parameters: NeighborsParameters, remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[(Long, Seq[NodeWithPosition])] = for { - // called here to ensure updates are applied - mapping <- getInfo(tracingId, version = None, remoteFallbackLayer) - agglomerateGraph <- getAgglomerateGraphForIdWithFallback(mapping, + agglomerateGraph <- getAgglomerateGraphForIdWithFallback(editableMappingInfo, tracingId, version, parameters.agglomerateId, @@ -674,6 +553,7 @@ class EditableMappingService @Inject()( neighborNodes } + /* def merge(newTracingId: String, tracingIds: List[String], remoteFallbackLayer: RemoteFallbackLayer)( implicit tc: TokenContext): Fox[Unit] = for { @@ -713,6 +593,8 @@ class EditableMappingService @Inject()( } } yield () + */ + private def batchRangeInclusive(from: Long, to: Long, batchSize: Long): Seq[(Long, Long)] = (0L to ((to - from) / batchSize)).map { batchIndex => val batchFrom = batchIndex * batchSize + from diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index bb33e48e366..5dc1e3017ba 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -10,7 +10,11 @@ import com.scalableminds.webknossos.datastore.SegmentToAgglomerateProto.{ SegmentToAgglomerateChunkProto } import com.scalableminds.webknossos.tracingstore.TSRemoteDatastoreClient -import com.scalableminds.webknossos.tracingstore.annotation.{RevertToVersionUpdateAction, UpdateAction} +import com.scalableminds.webknossos.tracingstore.annotation.{ + RevertToVersionUpdateAction, + TSAnnotationService, + UpdateAction +} import com.scalableminds.webknossos.tracingstore.tracings.volume.ReversionHelper import com.scalableminds.webknossos.tracingstore.tracings.{ KeyValueStoreImplicits, @@ -32,6 +36,7 @@ import scala.jdk.CollectionConverters.CollectionHasAsScala // this results in only one version increment in the db per update group class EditableMappingUpdater( + annotationId: String, tracingId: String, baseMappingName: String, oldVersion: Long, @@ -40,6 +45,7 @@ class EditableMappingUpdater( tokenContext: TokenContext, remoteDatastoreClient: TSRemoteDatastoreClient, editableMappingService: EditableMappingService, + annotationService: TSAnnotationService, tracingDataStore: TracingDataStore, relyOnAgglomerateIds: Boolean // False during merge and in case of multiple actions. Then, look up all agglomerate ids at positions ) extends KeyValueStoreImplicits @@ -418,7 +424,8 @@ class EditableMappingUpdater( implicit ec: ExecutionContext): Fox[EditableMappingInfo] = for { _ <- bool2Fox(revertAction.sourceVersion <= oldVersion) ?~> "trying to revert editable mapping to a version not yet present in the database" - oldInfo <- editableMappingService.getInfo(tracingId, Some(revertAction.sourceVersion), remoteFallbackLayer)( + oldInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId, Some(revertAction.sourceVersion))( + ec, tokenContext) _ = segmentToAgglomerateBuffer.clear() _ = agglomerateToGraphBuffer.clear() diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index c48e6ec3abe..e6a635c70d2 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -995,7 +995,7 @@ class VolumeTracingService @Inject()( remoteFallbackLayerFromVolumeTracing(tracingWithId._1, tracingWithId._2)) remoteFallbackLayer <- remoteFallbackLayers.headOption.toFox _ <- bool2Fox(remoteFallbackLayers.forall(_ == remoteFallbackLayer)) ?~> "Cannot merge editable mappings based on different dataset layers" - _ <- editableMappingService.merge(newTracingId, tracingsWithIds.map(_._2), remoteFallbackLayer) + // TODO_ <- editableMappingService.merge(newTracingId, tracingsWithIds.map(_._2), remoteFallbackLayer) } yield () } else if (tracingsWithIds.forall(tracingWithId => !tracingWithId._1.getHasEditableMapping)) { Fox.empty From 2a888999aee5ec5438552431645399d8c18d9e40 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 25 Sep 2024 09:23:04 +0200 Subject: [PATCH 079/150] construct editableMappingLayer already in annotationservice --- .../annotation/TSAnnotationService.scala | 19 ++++++ .../controllers/VolumeTracingController.scala | 14 +++-- ...VolumeTracingZarrStreamingController.scala | 9 ++- .../EditableMappingLayer.scala | 14 +++-- .../EditableMappingService.scala | 58 +++++++------------ .../tracings/volume/TSFullMeshService.scala | 9 ++- .../VolumeSegmentStatisticsService.scala | 28 ++++++--- .../volume/VolumeTracingService.scala | 34 +++++------ 8 files changed, 104 insertions(+), 81 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 0e77b6177e7..ff40b6ffa9d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -8,7 +8,9 @@ import com.scalableminds.webknossos.datastore.Annotation.AnnotationProto import com.scalableminds.webknossos.datastore.EditableMappingInfo.EditableMappingInfo import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing +import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ + EditableMappingLayer, EditableMappingService, EditableMappingUpdateAction, EditableMappingUpdater @@ -49,6 +51,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss tracingDataStore: TracingDataStore) extends KeyValueStoreImplicits with FallbackDataHelper + with ProtoGeometryImplicits with LazyLogging { def reportUpdates(annotationId: String, updateGroups: List[UpdateActionGroup])(implicit tc: TokenContext): Fox[Unit] = @@ -365,4 +368,20 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } ) } + + def editableMappingLayer(annotationId: String, tracingId: String, tracing: VolumeTracing)( + implicit tc: TokenContext): EditableMappingLayer = + EditableMappingLayer( + tracingId, + tracing.boundingBox, + resolutions = tracing.resolutions.map(vec3IntFromProto).toList, + largestSegmentId = Some(0L), + elementClass = tracing.elementClass, + tc, + tracing = tracing, + annotationId = annotationId, + tracingId = tracingId, + annotationService = this, + editableMappingService = editableMappingService + ) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index b8233ba03cd..98a3a903e61 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -170,9 +170,10 @@ class VolumeTracingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") - (data, indices) <- if (tracing.getHasEditableMapping) - editableMappingService.volumeData(tracing, tracingId, request.body) - else tracingService.data(tracingId, tracing, request.body) + (data, indices) <- if (tracing.getHasEditableMapping) { + val mappingLayer = annotationService.editableMappingLayer(annotationId, tracingId, tracing) + editableMappingService.volumeData(mappingLayer, request.body) + } else tracingService.data(tracingId, tracing, request.body) } yield Ok(data).withHeaders(getMissingBucketsHeaders(indices): _*) } } @@ -292,9 +293,10 @@ class VolumeTracingController @Inject()( // consecutive 3D points (i.e., nine floats) form a triangle. // There are no shared vertices between triangles. tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") - (vertices, neighbors) <- if (tracing.getHasEditableMapping) - editableMappingService.createAdHocMesh(tracing, tracingId, request.body) - else tracingService.createAdHocMesh(annotationId, tracingId, request.body) + (vertices: Array[Float], neighbors: List[Int]) <- if (tracing.getHasEditableMapping) { + val editableMappingLayer = annotationService.editableMappingLayer(annotationId, tracingId, tracing) + editableMappingService.createAdHocMesh(editableMappingLayer, request.body) + } else tracingService.createAdHocMesh(tracingId, tracing, request.body) } yield { // We need four bytes for each float val responseBuffer = ByteBuffer.allocate(vertices.length * 4).order(ByteOrder.LITTLE_ENDIAN) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala index b1f58de065d..66ff9cad10e 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala @@ -31,6 +31,7 @@ import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits import com.scalableminds.webknossos.datastore.models.{AdditionalCoordinate, WebknossosDataRequest} import com.scalableminds.webknossos.datastore.models.datasource.{AdditionalAxis, DataFormat, DataLayer, ElementClass} import com.scalableminds.webknossos.datastore.services.UserAccessRequest +import com.scalableminds.webknossos.tracingstore.annotation.TSAnnotationService import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeTracingService import com.scalableminds.webknossos.tracingstore.{ @@ -48,6 +49,7 @@ class VolumeTracingZarrStreamingController @Inject()( tracingService: VolumeTracingService, accessTokenService: TracingStoreAccessTokenService, editableMappingService: EditableMappingService, + annotationService: TSAnnotationService, remoteDataStoreClient: TSRemoteDatastoreClient, remoteWebknossosClient: TSRemoteWebknossosClient)(implicit ec: ExecutionContext) extends ExtendedController @@ -303,9 +305,10 @@ class VolumeTracingZarrStreamingController @Inject()( version = None, additionalCoordinates = additionalCoordinates ) - (data, missingBucketIndices) <- if (tracing.getHasEditableMapping) - editableMappingService.volumeData(tracing, tracingId, List(wkRequest)) - else tracingService.data(tracingId, tracing, List(wkRequest)) + (data, missingBucketIndices) <- if (tracing.getHasEditableMapping) { + val mappingLayer = annotationService.editableMappingLayer(annotationId, tracingId, tracing) + editableMappingService.volumeData(mappingLayer, List(wkRequest)) + } else tracingService.data(tracingId, tracing, List(wkRequest)) dataWithFallback <- getFallbackLayerDataIfEmpty(tracing, tracingId, data, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala index c46b0ffc44b..867e21192f5 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala @@ -22,6 +22,7 @@ import com.scalableminds.webknossos.datastore.models.datasource.{ import ucar.ma2.{Array => MultiArray} import com.scalableminds.webknossos.datastore.models.requests.DataReadInstruction import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService +import com.scalableminds.webknossos.tracingstore.annotation.TSAnnotationService import scala.concurrent.ExecutionContext @@ -35,10 +36,9 @@ class EditableMappingBucketProvider(layer: EditableMappingLayer) extends BucketP remoteFallbackLayer <- layer.editableMappingService .remoteFallbackLayerFromVolumeTracing(layer.tracing, layer.tracingId) // called here to ensure updates are applied - (editableMappingInfo, editableMappingVersion) <- layer.editableMappingService.getInfoAndActualVersion( - tracingId, - requestedVersion = None, - remoteFallbackLayer = remoteFallbackLayer)(layer.tokenContext) + editableMappingInfo <- layer.annotationService.getEditableMappingInfo(layer.annotationId, + tracingId, + Some(layer.version))(ec, layer.tokenContext) dataRequest: WebknossosDataRequest = WebknossosDataRequest( position = Vec3Int(bucket.topLeft.mag1X, bucket.topLeft.mag1Y, bucket.topLeft.mag1Z), mag = bucket.mag, @@ -56,7 +56,7 @@ class EditableMappingBucketProvider(layer: EditableMappingLayer) extends BucketP relevantMapping <- layer.editableMappingService.generateCombinedMappingForSegmentIds( segmentIds, editableMappingInfo, - editableMappingVersion, + layer.version, tracingId, remoteFallbackLayer)(layer.tokenContext) mappedData: Array[Byte] <- layer.editableMappingService.mapData(unmappedDataTyped, @@ -73,7 +73,9 @@ case class EditableMappingLayer(name: String, elementClass: ElementClass.Value, tokenContext: TokenContext, tracing: VolumeTracing, + annotationId: String, tracingId: String, + annotationService: TSAnnotationService, editableMappingService: EditableMappingService) extends SegmentationLayer { override val mags: List[MagLocator] = List.empty // MagLocators do not apply for annotation layers @@ -98,4 +100,6 @@ case class EditableMappingLayer(name: String, override def adminViewConfiguration: Option[LayerViewConfiguration] = None override def additionalAxes: Option[Seq[AdditionalAxis]] = None + + def version: Long = tracing.version } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index c48834d0992..779425710a2 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -217,13 +217,14 @@ class EditableMappingService @Inject()( voxelAsLong <- voxelAsLongArray.headOption } yield voxelAsLong - def volumeData(tracing: VolumeTracing, tracingId: String, dataRequests: DataRequestCollection)( - implicit tc: TokenContext): Fox[(Array[Byte], List[Int])] = { - - val dataLayer = editableMappingLayer(tracingId, tracing, tracingId) - val requests = dataRequests.map(r => - DataServiceDataRequest(null, dataLayer, r.cuboid(dataLayer), r.settings.copy(appliedAgglomerate = None))) - + def volumeData(editableMappingLayer: EditableMappingLayer, + dataRequests: DataRequestCollection): Fox[(Array[Byte], List[Int])] = { + val requests = dataRequests.map( + r => + DataServiceDataRequest(null, + editableMappingLayer, + r.cuboid(editableMappingLayer), + r.settings.copy(appliedAgglomerate = None))) binaryDataService.handleDataRequests(requests) } @@ -393,37 +394,20 @@ class EditableMappingService @Inject()( bytes = UnsignedIntegerArray.toByteArray(unsignedIntArray, elementClass) } yield bytes - private def editableMappingLayer(mappingName: String, tracing: VolumeTracing, tracingId: String)( - implicit tc: TokenContext): EditableMappingLayer = - EditableMappingLayer( - mappingName, - tracing.boundingBox, - resolutions = tracing.resolutions.map(vec3IntFromProto).toList, - largestSegmentId = Some(0L), - elementClass = tracing.elementClass, - tc, - tracing = tracing, - tracingId = tracingId, - editableMappingService = this + def createAdHocMesh(editableMappingLayer: EditableMappingLayer, + request: WebknossosAdHocMeshRequest): Fox[(Array[Float], List[Int])] = { + val adHocMeshRequest = AdHocMeshRequest( + dataSource = None, + dataLayer = editableMappingLayer, + cuboid = request.cuboid(editableMappingLayer), + segmentId = request.segmentId, + voxelSizeFactor = request.voxelSizeFactorInUnit, + mapping = None, + mappingType = None, + findNeighbors = request.findNeighbors ) - - def createAdHocMesh(tracing: VolumeTracing, tracingId: String, request: WebknossosAdHocMeshRequest)( - implicit tc: TokenContext): Fox[(Array[Float], List[Int])] = - for { - mappingName <- tracing.mappingName.toFox - segmentationLayer = editableMappingLayer(mappingName, tracing, tracingId) - adHocMeshRequest = AdHocMeshRequest( - dataSource = None, - dataLayer = segmentationLayer, - cuboid = request.cuboid(segmentationLayer), - segmentId = request.segmentId, - voxelSizeFactor = request.voxelSizeFactorInUnit, - mapping = None, - mappingType = None, - findNeighbors = request.findNeighbors - ) - result <- adHocMeshService.requestAdHocMeshViaActor(adHocMeshRequest) - } yield result + adHocMeshService.requestAdHocMeshViaActor(adHocMeshRequest) + } def getAgglomerateGraphForId(tracingId: String, version: Long, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala index 381f3420b5a..d51248458cd 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala @@ -16,6 +16,7 @@ import com.scalableminds.webknossos.datastore.models.{ WebknossosAdHocMeshRequest } import com.scalableminds.webknossos.datastore.services.{FullMeshHelper, FullMeshRequest} +import com.scalableminds.webknossos.tracingstore.annotation.TSAnnotationService import com.scalableminds.webknossos.tracingstore.tracings.FallbackDataHelper import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService import com.scalableminds.webknossos.tracingstore.{TSRemoteDatastoreClient, TSRemoteWebknossosClient} @@ -26,6 +27,7 @@ import scala.concurrent.ExecutionContext class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, editableMappingService: EditableMappingService, + annotationService: TSAnnotationService, volumeSegmentIndexService: VolumeSegmentIndexService, val remoteDatastoreClient: TSRemoteDatastoreClient, val remoteWebknossosClient: TSRemoteWebknossosClient) @@ -177,7 +179,8 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, adHocMeshRequest: WebknossosAdHocMeshRequest, annotationId: String, tracingId: String)(implicit tc: TokenContext): Fox[(Array[Float], List[Int])] = - if (tracing.getHasEditableMapping) - editableMappingService.createAdHocMesh(tracing, tracingId, adHocMeshRequest) - else volumeTracingService.createAdHocMesh(annotationId, tracingId, adHocMeshRequest) + if (tracing.getHasEditableMapping) { + val mappingLayer = annotationService.editableMappingLayer(annotationId, tracingId, tracing) + editableMappingService.createAdHocMesh(mappingLayer, adHocMeshRequest) + } else volumeTracingService.createAdHocMesh(tracingId, tracing, adHocMeshRequest) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentStatisticsService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentStatisticsService.scala index 6ad05f26680..82109a461a5 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentStatisticsService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentStatisticsService.scala @@ -10,12 +10,14 @@ import com.scalableminds.webknossos.datastore.models.{UnsignedInteger, UnsignedI import com.scalableminds.webknossos.datastore.models.datasource.DataLayer import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis +import com.scalableminds.webknossos.tracingstore.annotation.TSAnnotationService import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService import javax.inject.Inject import scala.concurrent.ExecutionContext class VolumeSegmentStatisticsService @Inject()(volumeTracingService: VolumeTracingService, + annotationService: TSAnnotationService, volumeSegmentIndexService: VolumeSegmentIndexService, editableMappingService: EditableMappingService) extends ProtoGeometryImplicits @@ -59,7 +61,12 @@ class VolumeSegmentStatisticsService @Inject()(volumeTracingService: VolumeTraci additionalCoordinates: Option[Seq[AdditionalCoordinate]])(implicit tc: TokenContext) = for { tracing <- volumeTracingService.find(annotationId, tracingId) ?~> "tracing.notFound" - bucketData <- getVolumeDataForPositions(tracing, tracingId, mag, Seq(bucketPosition), additionalCoordinates) + bucketData <- getVolumeDataForPositions(annotationId, + tracingId, + tracing, + mag, + Seq(bucketPosition), + additionalCoordinates) dataTyped: Array[UnsignedInteger] = UnsignedIntegerArray.fromByteArray( bucketData, elementClassFromProto(tracing.elementClass)) @@ -89,11 +96,13 @@ class VolumeSegmentStatisticsService @Inject()(volumeTracingService: VolumeTraci ) } yield allBucketPositions - private def getVolumeDataForPositions(tracing: VolumeTracing, - tracingId: String, - mag: Vec3Int, - bucketPositions: Seq[Vec3Int], - additionalCoordinates: Option[Seq[AdditionalCoordinate]])(implicit tc: TokenContext): Fox[Array[Byte]] = { + private def getVolumeDataForPositions( + annotationId: String, + tracingId: String, + tracing: VolumeTracing, + mag: Vec3Int, + bucketPositions: Seq[Vec3Int], + additionalCoordinates: Option[Seq[AdditionalCoordinate]])(implicit tc: TokenContext): Fox[Array[Byte]] = { val dataRequests = bucketPositions.map { position => WebknossosDataRequest( @@ -107,9 +116,10 @@ class VolumeSegmentStatisticsService @Inject()(volumeTracingService: VolumeTraci ) }.toList for { - (data, _) <- if (tracing.getHasEditableMapping) - editableMappingService.volumeData(tracing, tracingId, dataRequests) - else volumeTracingService.data(tracingId, tracing, dataRequests, includeFallbackDataIfAvailable = true) + (data, _) <- if (tracing.getHasEditableMapping) { + val mappingLayer = annotationService.editableMappingLayer(annotationId, tracingId, tracing) + editableMappingService.volumeData(mappingLayer, dataRequests) + } else volumeTracingService.data(tracingId, tracing, dataRequests, includeFallbackDataIfAvailable = true) } yield data } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index e6a635c70d2..f3ab813b6d3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -661,24 +661,22 @@ class VolumeTracingService @Inject()( def volumeBucketsAreEmpty(tracingId: String): Boolean = volumeDataStore.getMultipleKeys(None, Some(tracingId), limit = Some(1))(toBox).isEmpty - def createAdHocMesh(annotationId: String, tracingId: String, request: WebknossosAdHocMeshRequest)( - implicit tc: TokenContext): Fox[(Array[Float], List[Int])] = - for { - tracing <- find(annotationId: String, tracingId) ?~> "tracing.notFound" - segmentationLayer = volumeTracingLayer(tracingId, tracing, includeFallbackDataIfAvailable = true) - adHocMeshRequest = AdHocMeshRequest( - None, - segmentationLayer, - request.cuboid(segmentationLayer), - request.segmentId, - request.voxelSizeFactorInUnit, - None, - None, - request.additionalCoordinates, - request.findNeighbors - ) - result <- adHocMeshService.requestAdHocMeshViaActor(adHocMeshRequest) - } yield result + def createAdHocMesh(tracingId: String, tracing: VolumeTracing, request: WebknossosAdHocMeshRequest)( + implicit tc: TokenContext): Fox[(Array[Float], List[Int])] = { + val volumeLayer = volumeTracingLayer(tracingId, tracing, includeFallbackDataIfAvailable = true) + val adHocMeshRequest = AdHocMeshRequest( + None, + volumeLayer, + request.cuboid(volumeLayer), + request.segmentId, + request.voxelSizeFactorInUnit, + None, + None, + request.additionalCoordinates, + request.findNeighbors + ) + adHocMeshService.requestAdHocMeshViaActor(adHocMeshRequest) + } def findData(annotationId: String, tracingId: String)(implicit tc: TokenContext): Fox[Option[Vec3Int]] = for { From d57b4192fc50a32f3f3255904221786d6668dc69 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 25 Sep 2024 11:02:15 +0200 Subject: [PATCH 080/150] fix baseMappingName --- .../annotation/TSAnnotationService.scala | 9 ++++++++ .../controllers/VolumeTracingController.scala | 6 ++--- .../EditableMappingService.scala | 1 + .../tracings/volume/TSFullMeshService.scala | 22 ++++++++++--------- .../volume/VolumeTracingDownsampling.scala | 4 ++-- .../volume/VolumeTracingService.scala | 20 ++++++++--------- 6 files changed, 36 insertions(+), 26 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index ff40b6ffa9d..6cb417d0b86 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -384,4 +384,13 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss annotationService = this, editableMappingService = editableMappingService ) + + def baseMappingName(annotationId: String, tracingId: String, tracing: VolumeTracing)( + implicit ec: ExecutionContext, + tc: TokenContext): Fox[Option[String]] = + if (tracing.getHasEditableMapping) + for { + editableMappingInfo <- getEditableMappingInfo(annotationId, tracingId) + } yield Some(editableMappingInfo.baseMappingName) + else Fox.successful(tracing.mappingName) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 98a3a903e61..1679838400c 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -336,7 +336,7 @@ class VolumeTracingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) - mappingName <- tracingService.baseMappingName(tracing) + mappingName <- annotationService.baseMappingName(annotationId, tracingId, tracing) segmentVolumes <- Fox.serialCombined(request.body.segmentIds) { segmentId => volumeSegmentStatisticsService.getSegmentVolume(annotationId, tracingId, @@ -354,7 +354,7 @@ class VolumeTracingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { tracing <- tracingService.find(annotationId, tracingId) - mappingName <- tracingService.baseMappingName(tracing) + mappingName <- annotationService.baseMappingName(annotationId, tracingId, tracing) segmentBoundingBoxes: List[BoundingBox] <- Fox.serialCombined(request.body.segmentIds) { segmentId => volumeSegmentStatisticsService.getSegmentBoundingBox(annotationId, tracingId, @@ -373,7 +373,7 @@ class VolumeTracingController @Inject()( for { fallbackLayer <- tracingService.getFallbackLayer(annotationId, tracingId) tracing <- tracingService.find(annotationId, tracingId) - mappingName <- tracingService.baseMappingName(tracing) + mappingName <- annotationService.baseMappingName(annotationId, tracingId, tracing) _ <- bool2Fox(DataLayer.bucketSize <= request.body.cubeSize) ?~> "cubeSize must be at least one bucket (32³)" bucketPositionsRaw: ListOfVec3IntProto <- volumeSegmentIndexService .getSegmentToBucketIndexWithEmptyFallbackWithoutBuffer( diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index 779425710a2..b8ea620e1b3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -195,6 +195,7 @@ class EditableMappingService @Inject()( bool2Fox(tracing.getHasEditableMapping) ?~> "annotation.volume.noEditableMapping" def getBaseMappingName(tracingId: String): Fox[Option[String]] = + // TODO Fox.successful(None) def findSegmentIdAtPositionIfNeeded(remoteFallbackLayer: RemoteFallbackLayer, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala index d51248458cd..541b0ed519a 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala @@ -42,16 +42,18 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, for { tracing <- volumeTracingService.find(annotationId, tracingId) ?~> "tracing.notFound" data <- if (fullMeshRequest.meshFileName.isDefined) - loadFullMeshFromMeshfile(tracing, tracingId, fullMeshRequest) - else loadFullMeshFromAdHoc(tracing, annotationId, tracingId, fullMeshRequest) + loadFullMeshFromMeshfile(annotationId, tracingId, tracing, fullMeshRequest) + else loadFullMeshFromAdHoc(annotationId, tracingId, tracing, fullMeshRequest) } yield data - private def loadFullMeshFromMeshfile(tracing: VolumeTracing, tracingId: String, fullMeshRequest: FullMeshRequest)( - implicit ec: ExecutionContext, - tc: TokenContext): Fox[Array[Byte]] = + private def loadFullMeshFromMeshfile( + annotationId: String, + tracingId: String, + tracing: VolumeTracing, + fullMeshRequest: FullMeshRequest)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Array[Byte]] = for { remoteFallbackLayer <- remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - baseMappingName <- volumeTracingService.baseMappingName(tracing) + baseMappingName <- annotationService.baseMappingName(annotationId, tracingId, tracing) fullMeshRequestAdapted = if (tracing.getHasEditableMapping) fullMeshRequest.copy(mappingName = baseMappingName, editableMappingTracingId = Some(tracingId), @@ -61,9 +63,9 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, } yield array private def loadFullMeshFromAdHoc( - tracing: VolumeTracing, annotationId: String, tracingId: String, + tracing: VolumeTracing, fullMeshRequest: FullMeshRequest)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Array[Byte]] = for { mag <- fullMeshRequest.mag.toFox ?~> "mag.neededForAdHoc" @@ -71,7 +73,7 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, before = Instant.now voxelSize <- remoteDatastoreClient.voxelSizeForTracingWithCache(tracingId) ?~> "voxelSize.failedToFetch" verticesForChunks <- if (tracing.hasSegmentIndex.getOrElse(false)) - getAllAdHocChunksWithSegmentIndex(annotationId, tracing, tracingId, mag, voxelSize, fullMeshRequest) + getAllAdHocChunksWithSegmentIndex(annotationId, tracingId, tracing, mag, voxelSize, fullMeshRequest) else getAllAdHocChunksWithNeighborLogic( tracing, @@ -90,14 +92,14 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, private def getAllAdHocChunksWithSegmentIndex( annotationId: String, - tracing: VolumeTracing, tracingId: String, + tracing: VolumeTracing, mag: Vec3Int, voxelSize: VoxelSize, fullMeshRequest: FullMeshRequest)(implicit ec: ExecutionContext, tc: TokenContext): Fox[List[Array[Float]]] = for { fallbackLayer <- volumeTracingService.getFallbackLayer(annotationId, tracingId) - mappingName <- volumeTracingService.baseMappingName(tracing) + mappingName <- annotationService.baseMappingName(annotationId, tracingId, tracing) bucketPositionsRaw: ListOfVec3IntProto <- volumeSegmentIndexService .getSegmentToBucketIndexWithEmptyFallbackWithoutBuffer( fallbackLayer, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala index 97cf49ed688..5a85c53a8f8 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala @@ -74,7 +74,7 @@ trait VolumeTracingDownsampling protected def editableMappingTracingId(tracing: VolumeTracing, tracingId: String): Option[String] - protected def baseMappingName(tracing: VolumeTracing): Fox[Option[String]] + protected def selectMappingName(tracing: VolumeTracing): Fox[Option[String]] protected def volumeSegmentIndexClient: FossilDBClient @@ -119,7 +119,7 @@ trait VolumeTracingDownsampling _ <- Fox.serialCombined(updatedBucketsMutable.toList) { bucketPosition: BucketPosition => for { _ <- saveBucket(dataLayer, bucketPosition, bucketDataMapMutable(bucketPosition), tracing.version) - mappingName <- baseMappingName(tracing) + mappingName <- selectMappingName(tracing) _ <- Fox.runIfOptionTrue(tracing.hasSegmentIndex)( updateSegmentIndex( segmentIndexBuffer, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index f3ab813b6d3..31b40df0961 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -26,7 +26,6 @@ import com.scalableminds.webknossos.datastore.services._ import com.scalableminds.webknossos.tracingstore.annotation.{TSAnnotationService, UpdateActionGroup} import com.scalableminds.webknossos.tracingstore.tracings.TracingType.TracingType import com.scalableminds.webknossos.tracingstore.tracings._ -import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeDataZipFormat.VolumeDataZipFormat import com.scalableminds.webknossos.tracingstore.{ TSRemoteDatastoreClient, @@ -57,7 +56,6 @@ class VolumeTracingService @Inject()( implicit val ec: ExecutionContext, val handledGroupIdStore: TracingStoreRedisStore, val uncommittedUpdatesStore: TracingStoreRedisStore, - editableMappingService: EditableMappingService, val temporaryTracingIdStore: TracingStoreRedisStore, val remoteDatastoreClient: TSRemoteDatastoreClient, val annotationService: TSAnnotationService, @@ -163,7 +161,7 @@ class VolumeTracingService @Inject()( dataLayer = volumeTracingLayer(tracingId, volumeTracing) actionBucketData <- action.base64Data.map(Base64.getDecoder.decode).toFox _ <- saveBucket(dataLayer, bucketPosition, actionBucketData, updateGroupVersion) ?~> "failed to save bucket" - mappingName <- baseMappingName(volumeTracing) + mappingName <- selectMappingName(volumeTracing) _ <- Fox.runIfOptionTrue(volumeTracing.hasSegmentIndex) { for { previousBucketBytes <- loadBucket(dataLayer, bucketPosition, Some(updateGroupVersion - 1L)).futureBox @@ -197,9 +195,9 @@ class VolumeTracingService @Inject()( override def editableMappingTracingId(tracing: VolumeTracing, tracingId: String): Option[String] = if (tracing.getHasEditableMapping) Some(tracingId) else None - override def baseMappingName(tracing: VolumeTracing): Fox[Option[String]] = + def selectMappingName(tracing: VolumeTracing): Fox[Option[String]] = if (tracing.getHasEditableMapping) - tracing.mappingName.map(editableMappingService.getBaseMappingName).getOrElse(Fox.successful(None)) + Fox.failure("mappingName called on volumeTracing with editableMapping!") else Fox.successful(tracing.mappingName) private def deleteSegmentData(annotationId: String, @@ -217,7 +215,7 @@ class VolumeTracingService @Inject()( } else { possibleAdditionalCoordinates.toList } - mappingName <- baseMappingName(volumeTracing) + mappingName <- selectMappingName(volumeTracing) _ <- Fox.serialCombined(volumeTracing.resolutions.toList)(resolution => Fox.serialCombined(additionalCoordinateList)(additionalCoordinates => { val mag = vec3IntFromProto(resolution) @@ -350,7 +348,7 @@ class VolumeTracingService @Inject()( _ = if (resolutionSet.nonEmpty) resolutionSets.add(resolutionSet) } yield () } - mappingName <- baseMappingName(tracing) + mappingName <- selectMappingName(tracing) resolutions <- // if none of the tracings contained any volume data do not save buckets, use full resolution list, as already initialized on wk-side if (resolutionSets.isEmpty) @@ -416,7 +414,7 @@ class VolumeTracingService @Inject()( val savedResolutions = new mutable.HashSet[Vec3Int]() for { fallbackLayer <- getFallbackLayer(annotationId, tracingId) - mappingName <- baseMappingName(tracing) + mappingName <- selectMappingName(tracing) segmentIndexBuffer = new VolumeSegmentIndexBuffer( tracingId, volumeSegmentIndexClient, @@ -578,7 +576,7 @@ class VolumeTracingService @Inject()( AdditionalAxis.fromProtosAsOpt(sourceTracing.additionalAxes), tc ) - mappingName <- baseMappingName(sourceTracing) + mappingName <- selectMappingName(sourceTracing) _ <- Fox.serialCombined(buckets) { case (bucketPosition, bucketData) => if (destinationTracing.resolutions.contains(vec3IntToProto(bucketPosition.mag))) { @@ -862,7 +860,7 @@ class VolumeTracingService @Inject()( sourceDataLayer = volumeTracingLayer(tracingId, tracing, isTemporaryTracing) buckets: Iterator[(BucketPosition, Array[Byte])] = sourceDataLayer.bucketProvider.bucketStream() fallbackLayer <- getFallbackLayer(annotationId, tracingId) - mappingName <- baseMappingName(tracing) + mappingName <- selectMappingName(tracing) segmentIndexBuffer = new VolumeSegmentIndexBuffer(tracingId, volumeSegmentIndexClient, currentVersion + 1L, @@ -938,7 +936,7 @@ class VolumeTracingService @Inject()( tracing.elementClass) dataLayer = volumeTracingLayer(tracingId, tracing) fallbackLayer <- getFallbackLayer(annotationId, tracingId) - mappingName <- baseMappingName(tracing) + mappingName <- selectMappingName(tracing) segmentIndexBuffer <- Fox.successful( new VolumeSegmentIndexBuffer(tracingId, volumeSegmentIndexClient, From 401c7f95580c29300ad50378fd92c684b62a1f12 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 25 Sep 2024 11:41:31 +0200 Subject: [PATCH 081/150] wip: fix segmentsForAgglomerate --- .../WKRemoteTracingStoreController.scala | 10 ++++++++++ conf/webknossos.latest.routes | 1 + .../tracingstore/TSRemoteWebknossosClient.scala | 13 +++++++++++++ .../controllers/EditableMappingController.scala | 6 ++++-- ...com.scalableminds.webknossos.tracingstore.routes | 2 +- 5 files changed, 29 insertions(+), 3 deletions(-) diff --git a/app/controllers/WKRemoteTracingStoreController.scala b/app/controllers/WKRemoteTracingStoreController.scala index 2931937f09d..1476dba7c28 100644 --- a/app/controllers/WKRemoteTracingStoreController.scala +++ b/app/controllers/WKRemoteTracingStoreController.scala @@ -114,6 +114,16 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore } } + def annotationIdForTracing(name: String, key: String, tracingId: String): Action[AnyContent] = + Action.async { implicit request => + tracingStoreService.validateAccess(name, key) { _ => + implicit val ctx: DBAccessContext = GlobalAccessContext + for { + annotation <- annotationInformationProvider.annotationForTracing(tracingId) ?~> s"No annotation for tracing $tracingId" + } yield Ok(Json.toJson(annotation._id)) + } + } + def dataStoreUriForDataset(name: String, key: String, organizationId: Option[String], diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes index c250497ba6c..5b9ad059207 100644 --- a/conf/webknossos.latest.routes +++ b/conf/webknossos.latest.routes @@ -125,6 +125,7 @@ POST /tracingstores/:name/validateUserAccess PUT /tracingstores/:name controllers.TracingStoreController.update(name: String) GET /tracingstores/:name/dataSource controllers.WKRemoteTracingStoreController.dataSourceForTracing(name: String, key: String, tracingId: String) GET /tracingstores/:name/dataSourceId controllers.WKRemoteTracingStoreController.dataSourceIdForTracing(name: String, key: String, tracingId: String) +GET /tracingstores/:name/annotationId controllers.WKRemoteTracingStoreController.annotationIdForTracing(name: String, key: String, tracingId: String) GET /tracingstores/:name/dataStoreUri/:datasetName controllers.WKRemoteTracingStoreController.dataStoreUriForDataset(name: String, key: String, organizationId: Option[String], datasetName: String) # User access tokens for datastore authentication diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala index 09e3e019e8c..08198fe4ff7 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala @@ -19,6 +19,7 @@ import play.api.libs.json.{JsObject, Json, OFormat} import play.api.libs.ws.WSResponse import scala.concurrent.ExecutionContext +import scala.concurrent.duration.DurationInt case class TracingUpdatesReport(annotationId: String, // TODO stats per tracing id? @@ -44,6 +45,7 @@ class TSRemoteWebknossosClient @Inject()( private val webknossosUri: String = config.Tracingstore.WebKnossos.uri private lazy val dataSourceIdByTracingIdCache: AlfuCache[String, DataSourceId] = AlfuCache() + private lazy val annotationIdByTracingIdCache: AlfuCache[String, String] = AlfuCache(timeToLive = 5 minutes) def reportTracingUpdates(tracingUpdatesReport: TracingUpdatesReport): Fox[WSResponse] = rpc(s"$webknossosUri/api/tracingstores/$tracingStoreName/handleTracingUpdateReport") @@ -74,6 +76,17 @@ class TSRemoteWebknossosClient @Inject()( .getWithJsonResponse[DataSourceId] ) + // TODO what about temporary/compound tracings? + def getAnnotationIdForTracing(tracingId: String)(implicit ec: ExecutionContext): Fox[String] = + annotationIdByTracingIdCache.getOrLoad( + tracingId, + tracingId => + rpc(s"$webknossosUri/api/tracingstores/$tracingStoreName/annotationId") + .addQueryString("tracingId" -> tracingId) + .addQueryString("key" -> tracingStoreKey) + .getWithJsonResponse[String] + ) + override def requestUserAccess(accessRequest: UserAccessRequest)(implicit tc: TokenContext): Fox[UserAccessAnswer] = rpc(s"$webknossosUri/api/tracingstores/$tracingStoreName/validateUserAccess") .addQueryString("key" -> tracingStoreKey) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala index 24e41a83f88..cf095dbd421 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala @@ -7,7 +7,7 @@ import com.scalableminds.webknossos.datastore.ListOfLong.ListOfLong import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.controllers.Controller import com.scalableminds.webknossos.datastore.services.{EditableMappingSegmentListResult, UserAccessRequest} -import com.scalableminds.webknossos.tracingstore.TracingStoreAccessTokenService +import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingStoreAccessTokenService} import com.scalableminds.webknossos.tracingstore.annotation.{ AnnotationTransactionService, TSAnnotationService, @@ -27,6 +27,7 @@ import scala.concurrent.ExecutionContext class EditableMappingController @Inject()(volumeTracingService: VolumeTracingService, annotationService: TSAnnotationService, + remoteWebknossosClient: TSRemoteWebknossosClient, accessTokenService: TracingStoreAccessTokenService, editableMappingService: EditableMappingService, annotationTransactionService: AnnotationTransactionService)( @@ -117,11 +118,12 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer } } - def segmentIdsForAgglomerate(annotationId: String, tracingId: String, agglomerateId: Long): Action[AnyContent] = + def segmentIdsForAgglomerate(tracingId: String, agglomerateId: Long): Action[AnyContent] = Action.async { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- volumeTracingService.find(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index a7b9619e365..8d074e46e96 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -36,7 +36,7 @@ POST /volume/mergedFromContents @c # Editable Mappings POST /mapping/:annotationId/:tracingId/makeMappingEditable @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.makeMappingEditable(annotationId: String, tracingId: String) GET /mapping/:annotationId/:tracingId/info @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.editableMappingInfo(annotationId: String, tracingId: String, version: Option[Long]) -GET /mapping/:annotationId/:tracingId/segmentsForAgglomerate @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.segmentIdsForAgglomerate(annotationId: String, tracingId: String, agglomerateId: Long) +GET /mapping/:tracingId/segmentsForAgglomerate @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.segmentIdsForAgglomerate(tracingId: String, agglomerateId: Long) POST /mapping/:annotationId/:tracingId/agglomeratesForSegments @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateIdsForSegments(annotationId: String, tracingId: String) POST /mapping/:annotationId/:tracingId/agglomerateGraphMinCut @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateGraphMinCut(annotationId: String, tracingId: String) POST /mapping/:annotationId/:tracingId/agglomerateGraphNeighbors @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateGraphNeighbors(annotationId: String, tracingId: String) From 5f71bfc3a8d234339a7e4af86bcc997f8807cfe6 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 25 Sep 2024 13:22:02 +0200 Subject: [PATCH 082/150] flush --- .../annotation/AnnotationWithTracings.scala | 18 +++++++++++ .../annotation/TSAnnotationService.scala | 30 +++++++++++++++++-- .../tracings/volume/VolumeUpdateActions.scala | 5 ++-- 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index e8c59a7010d..37b17dfb8f5 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -31,6 +31,24 @@ case class AnnotationWithTracings( } } yield skeletonTracing + def getVolumes: List[(String, VolumeTracing)] = + tracingsById.view.flatMap { + case (id, Right(vt: VolumeTracing)) => Some(id, vt) + case _ => None + }.toList + + def getSkeletons: List[(String, SkeletonTracing)] = + tracingsById.view.flatMap { + case (id, Left(st: SkeletonTracing)) => Some(id, st) + case _ => None + }.toList + + def getEditableMappingsInfo: List[(String, EditableMappingInfo)] = + editableMappingsByTracingId.view.flatMap { + case (id, (info: EditableMappingInfo, _)) => Some(id, info) + case _ => None + }.toList + def getVolume(tracingId: String): Box[VolumeTracing] = for { tracingEither <- tracingsById.get(tracingId) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 6cb417d0b86..6f3a995bcb8 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -322,12 +322,36 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss else { for { updated <- updateIter(Some(annotation), updates) - _ <- updated.flushBufferedUpdates() - // todo: save materialized tracings + editable mapping info - } yield updated.withVersion(targetVersion) + updatedWithNewVerson = updated.withVersion(targetVersion) + _ <- updatedWithNewVerson.flushBufferedUpdates() + _ <- flushUpdatedTracings(updatedWithNewVerson) + _ <- flushAnnotationInfo(annotationId, updatedWithNewVerson) + } yield updatedWithNewVerson } } + private def flushUpdatedTracings(annotationWithTracings: AnnotationWithTracings)(implicit ec: ExecutionContext) = + // TODO skip some flushes to save disk space (e.g. skeletons only nth version, or only if requested?) + for { + _ <- Fox.serialCombined(annotationWithTracings.getVolumes) { + case (volumeTracingId, volumeTracing) => + tracingDataStore.volumes.put(volumeTracingId, volumeTracing.version, volumeTracing) + } + _ <- Fox.serialCombined(annotationWithTracings.getSkeletons) { + case (skeletonTracingId, skeletonTracing: SkeletonTracing) => + tracingDataStore.skeletons.put(skeletonTracingId, skeletonTracing.version, skeletonTracing) + } + _ <- Fox.serialCombined(annotationWithTracings.getEditableMappingsInfo) { + case (volumeTracingId, editableMappingInfo) => + tracingDataStore.editableMappingsInfo.put(volumeTracingId, + annotationWithTracings.version, + editableMappingInfo) + } + } yield () + + private def flushAnnotationInfo(annotationId: String, annotationWithTracings: AnnotationWithTracings) = + tracingDataStore.annotations.put(annotationId, annotationWithTracings.version, annotationWithTracings.annotation) + private def determineTargetVersion(annotation: AnnotationProto, annotationId: String, targetVersionOpt: Option[Long]): Fox[Long] = diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala index f2d791d2973..04833f3dbcc 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala @@ -269,8 +269,7 @@ case class DeleteSegmentVolumeAction(id: Long, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends BucketMutatingVolumeUpdateAction - with ApplyableVolumeUpdateAction { + extends ApplyableVolumeUpdateAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) @@ -288,7 +287,7 @@ case class DeleteSegmentDataVolumeAction(id: Long, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) - extends VolumeUpdateAction { + extends BucketMutatingVolumeUpdateAction { override def addTimestamp(timestamp: Long): VolumeUpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) From 7f4a575228355ffc38cc6c779f4b9ded21020479 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 26 Sep 2024 13:42:05 +0200 Subject: [PATCH 083/150] look up annotation id by tracing id rather than expanding the routes --- conf/messages | 1 + frontend/javascripts/admin/admin_rest_api.ts | 19 +-- .../bucket_data_handling/wkstore_adapter.ts | 3 +- .../oxalis/model/sagas/mapping_saga.ts | 1 - .../oxalis/model/sagas/proofread_saga.ts | 14 +- .../oxalis/model_initialization.ts | 4 +- .../segments_tab/segments_view_helper.tsx | 3 +- .../TSRemoteWebknossosClient.scala | 5 +- .../EditableMappingController.scala | 18 ++- .../SkeletonTracingController.scala | 4 +- .../controllers/TracingController.scala | 3 +- .../controllers/VolumeTracingController.scala | 44 ++++--- ...VolumeTracingZarrStreamingController.scala | 43 +++--- .../tracings/TracingService.scala | 14 +- .../skeleton/SkeletonTracingService.scala | 4 +- ...alableminds.webknossos.tracingstore.routes | 123 +++++++++--------- 16 files changed, 148 insertions(+), 155 deletions(-) diff --git a/conf/messages b/conf/messages index be5596c7627..191c8e6627c 100644 --- a/conf/messages +++ b/conf/messages @@ -252,6 +252,7 @@ annotation.deleteLayer.explorationalsOnly=Could not delete a layer because it is annotation.deleteLayer.onlyLayer=Could not delete layer because it is the only layer in this annotation. annotation.layer.notFound=Layer could not be found. annotation.getNewestVersion.failed=Could not get the newest version information for this annotation layer +annotation.idForTracing.failed=Could not find the annotation id for this tracing id. mesh.notFound=Mesh could not be found mesh.write.failed=Failed to convert mesh info to json diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts index 9e3e4ebda6a..89d6c5fe1cf 100644 --- a/frontend/javascripts/admin/admin_rest_api.ts +++ b/frontend/javascripts/admin/admin_rest_api.ts @@ -896,7 +896,7 @@ export async function getTracingForAnnotationType( const possibleVersionString = version != null ? `&version=${version}` : ""; const tracingArrayBuffer = await doWithToken((token) => Request.receiveArraybuffer( - `${annotation.tracingStore.url}/tracings/${tracingType}/${annotation.id}/${tracingId}?token=${token}${possibleVersionString}`, + `${annotation.tracingStore.url}/tracings/${tracingType}/${tracingId}?token=${token}${possibleVersionString}`, { headers: { Accept: "application/x-protobuf", @@ -1614,12 +1614,11 @@ export function fetchMapping( export function makeMappingEditable( tracingStoreUrl: string, - annotationId: string, tracingId: string, ): Promise { return doWithToken((token) => Request.receiveJSON( - `${tracingStoreUrl}/tracings/mapping/${annotationId}/${tracingId}/makeMappingEditable?token=${token}`, + `${tracingStoreUrl}/tracings/mapping/${tracingId}/makeMappingEditable?token=${token}`, { method: "POST", }, @@ -1629,13 +1628,10 @@ export function makeMappingEditable( export function getEditableMappingInfo( tracingStoreUrl: string, - annotationId: string, tracingId: string, ): Promise { return doWithToken((token) => - Request.receiveJSON( - `${tracingStoreUrl}/tracings/mapping/${annotationId}/${tracingId}/info?token=${token}`, - ), + Request.receiveJSON(`${tracingStoreUrl}/tracings/mapping/${tracingId}/info?token=${token}`), ); } @@ -2122,7 +2118,6 @@ export async function getAgglomeratesForSegmentsFromDatastore( tracingStoreUrl: string, - annotationId: string, tracingId: string, segmentIds: Array, ): Promise { @@ -2132,7 +2127,7 @@ export async function getAgglomeratesForSegmentsFromTracingstore Request.receiveArraybuffer( - `${tracingStoreUrl}/tracings/mapping/${annotationId}/${tracingId}/agglomeratesForSegments?token=${token}`, + `${tracingStoreUrl}/tracings/mapping/${tracingId}/agglomeratesForSegments?token=${token}`, { method: "POST", body: segmentIdBuffer, @@ -2311,7 +2306,6 @@ type MinCutTargetEdge = { }; export async function getEdgesForAgglomerateMinCut( tracingStoreUrl: string, - annotationId: string, tracingId: string, segmentsInfo: { segmentId1: NumberLike; @@ -2323,7 +2317,7 @@ export async function getEdgesForAgglomerateMinCut( ): Promise> { return doWithToken((token) => Request.sendJSONReceiveJSON( - `${tracingStoreUrl}/tracings/mapping/${annotationId}/${tracingId}/agglomerateGraphMinCut?token=${token}`, + `${tracingStoreUrl}/tracings/mapping/${tracingId}/agglomerateGraphMinCut?token=${token}`, { data: { ...segmentsInfo, @@ -2345,7 +2339,6 @@ export type NeighborInfo = { export async function getNeighborsForAgglomerateNode( tracingStoreUrl: string, tracingId: string, - annotationId: string, segmentInfo: { segmentId: NumberLike; mag: Vector3; @@ -2355,7 +2348,7 @@ export async function getNeighborsForAgglomerateNode( ): Promise { return doWithToken((token) => Request.sendJSONReceiveJSON( - `${tracingStoreUrl}/tracings/mapping/${annotationId}/${tracingId}/agglomerateGraphNeighbors?token=${token}`, + `${tracingStoreUrl}/tracings/mapping/${tracingId}/agglomerateGraphNeighbors?token=${token}`, { data: { ...segmentInfo, diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.ts b/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.ts index 963ca797e1c..4db8385e75b 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.ts +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.ts @@ -110,8 +110,7 @@ export async function requestWithFallback( optLayerName || layerInfo.name }`; - const getTracingStoreUrl = () => - `${tracingStoreHost}/tracings/volume/${state.tracing.annotationId}/${layerInfo.name}`; + const getTracingStoreUrl = () => `${tracingStoreHost}/tracings/volume/${layerInfo.name}`; const maybeVolumeTracing = "tracingId" in layerInfo && layerInfo.tracingId != null diff --git a/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts b/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts index ddfb14597ca..1a0044c4370 100644 --- a/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/mapping_saga.ts @@ -455,7 +455,6 @@ function* updateLocalHdf5Mapping( ? yield* call( getAgglomeratesForSegmentsFromTracingstore, annotation.tracingStore.url, - annotation.annotationId, editableMapping.tracingId, Array.from(newSegmentIds), ) diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts index 653f5f5919a..de523f53795 100644 --- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts @@ -265,7 +265,6 @@ function* createEditableMapping(): Saga { * name of the HDF5 mapping for which the editable mapping is about to be created. */ const tracingStoreUrl = yield* select((state) => state.tracing.tracingStore.url); - const annotationId = yield* select((state) => state.tracing.annotationId); // Save before making the mapping editable to make sure the correct mapping is activated in the backend yield* call([Model, Model.ensureSavedState]); // Get volume tracing again to make sure the version is up to date @@ -276,12 +275,7 @@ function* createEditableMapping(): Saga { const volumeTracingId = upToDateVolumeTracing.tracingId; const layerName = volumeTracingId; - const serverEditableMapping = yield* call( - makeMappingEditable, - tracingStoreUrl, - annotationId, - volumeTracingId, - ); + const serverEditableMapping = yield* call(makeMappingEditable, tracingStoreUrl, volumeTracingId); // The server increments the volume tracing's version by 1 when switching the mapping to an editable one yield* put(setVersionNumberAction(upToDateVolumeTracing.version + 1, "volume", volumeTracingId)); yield* put(setMappingNameAction(layerName, volumeTracingId, "HDF5")); @@ -546,7 +540,6 @@ function* performMinCut( } const tracingStoreUrl = yield* select((state) => state.tracing.tracingStore.url); - const annotationId = yield* select((state) => state.tracing.annotationId); const segmentsInfo = { segmentId1: sourceSegmentId, segmentId2: targetSegmentId, @@ -558,7 +551,6 @@ function* performMinCut( const edgesToRemove = yield* call( getEdgesForAgglomerateMinCut, tracingStoreUrl, - annotationId, volumeTracingId, segmentsInfo, ); @@ -609,7 +601,6 @@ function* performCutFromNeighbors( { didCancel: false; neighborInfo: NeighborInfo } | { didCancel: true; neighborInfo?: null } > { const tracingStoreUrl = yield* select((state) => state.tracing.tracingStore.url); - const annotationId = yield* select((state) => state.tracing.annotationId); const segmentsInfo = { segmentId, mag: agglomerateFileMag, @@ -620,7 +611,6 @@ function* performCutFromNeighbors( const neighborInfo = yield* call( getNeighborsForAgglomerateNode, tracingStoreUrl, - annotationId, volumeTracingId, segmentsInfo, ); @@ -1283,13 +1273,11 @@ function* splitAgglomerateInMapping( .map(([segmentId, _agglomerateId]) => segmentId); const tracingStoreUrl = yield* select((state) => state.tracing.tracingStore.url); - const annotationId = yield* select((state) => state.tracing.annotationId); // Ask the server to map the (split) segment ids. This creates a partial mapping // that only contains these ids. const mappingAfterSplit = yield* call( getAgglomeratesForSegmentsFromTracingstore, tracingStoreUrl, - annotationId, volumeTracingId, splitSegmentIds, ); diff --git a/frontend/javascripts/oxalis/model_initialization.ts b/frontend/javascripts/oxalis/model_initialization.ts index adb09da6fb5..5ff7619bfab 100644 --- a/frontend/javascripts/oxalis/model_initialization.ts +++ b/frontend/javascripts/oxalis/model_initialization.ts @@ -213,7 +213,6 @@ export async function initialize( if (annotation != null) { const editableMappings = await fetchEditableMappings( annotation.tracingStore.url, - annotation.id, serverVolumeTracings, ); initializeTracing(annotation, serverTracings, editableMappings); @@ -249,12 +248,11 @@ async function fetchParallel( async function fetchEditableMappings( tracingStoreUrl: string, - annotationId: string, serverVolumeTracings: ServerVolumeTracing[], ): Promise { const promises = serverVolumeTracings .filter((tracing) => tracing.hasEditableMapping) - .map((tracing) => getEditableMappingInfo(tracingStoreUrl, annotationId, tracing.id)); + .map((tracing) => getEditableMappingInfo(tracingStoreUrl, tracing.id)); return Promise.all(promises); } diff --git a/frontend/javascripts/oxalis/view/right-border-tabs/segments_tab/segments_view_helper.tsx b/frontend/javascripts/oxalis/view/right-border-tabs/segments_tab/segments_view_helper.tsx index 818b990e681..0440554ef95 100644 --- a/frontend/javascripts/oxalis/view/right-border-tabs/segments_tab/segments_view_helper.tsx +++ b/frontend/javascripts/oxalis/view/right-border-tabs/segments_tab/segments_view_helper.tsx @@ -50,8 +50,7 @@ export function getVolumeRequestUrl( return `${dataset.dataStore.url}/data/datasets/${dataset.owningOrganization}/${dataset.name}/layers/${visibleSegmentationLayer.name}`; } else { const tracingStoreHost = tracing?.tracingStore.url; - const annotationId = tracing?.annotationId; - return `${tracingStoreHost}/tracings/volume/${annotationId}/${tracingId}`; + return `${tracingStoreHost}/tracings/volume/${tracingId}`; } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala index 08198fe4ff7..8024af01d16 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala @@ -45,7 +45,8 @@ class TSRemoteWebknossosClient @Inject()( private val webknossosUri: String = config.Tracingstore.WebKnossos.uri private lazy val dataSourceIdByTracingIdCache: AlfuCache[String, DataSourceId] = AlfuCache() - private lazy val annotationIdByTracingIdCache: AlfuCache[String, String] = AlfuCache(timeToLive = 5 minutes) + private lazy val annotationIdByTracingIdCache: AlfuCache[String, String] = + AlfuCache(maxCapacity = 10000, timeToLive = 5 minutes) def reportTracingUpdates(tracingUpdatesReport: TracingUpdatesReport): Fox[WSResponse] = rpc(s"$webknossosUri/api/tracingstores/$tracingStoreName/handleTracingUpdateReport") @@ -85,7 +86,7 @@ class TSRemoteWebknossosClient @Inject()( .addQueryString("tracingId" -> tracingId) .addQueryString("key" -> tracingStoreKey) .getWithJsonResponse[String] - ) + ) ?~> "annotation.idForTracing.failed" override def requestUserAccess(accessRequest: UserAccessRequest)(implicit tc: TokenContext): Fox[UserAccessAnswer] = rpc(s"$webknossosUri/api/tracingstores/$tracingStoreName/validateUserAccess") diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala index cf095dbd421..cc781fe939d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala @@ -35,11 +35,12 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer bodyParsers: PlayBodyParsers) extends Controller { - def makeMappingEditable(annotationId: String, tracingId: String): Action[AnyContent] = + def makeMappingEditable(tracingId: String): Action[AnyContent] = Action.async { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- volumeTracingService.find(annotationId, tracingId) tracingMappingName <- tracing.mappingName ?~> "annotation.noMappingSet" _ <- assertMappingIsNotLocked(tracing) @@ -104,11 +105,12 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer } */ - def editableMappingInfo(annotationId: String, tracingId: String, version: Option[Long]): Action[AnyContent] = + def editableMappingInfo(tracingId: String, version: Option[Long]): Action[AnyContent] = Action.async { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- volumeTracingService.find(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) editableMappingInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId, version) @@ -141,11 +143,12 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer } } - def agglomerateIdsForSegments(annotationId: String, tracingId: String): Action[ListOfLong] = + def agglomerateIdsForSegments(tracingId: String): Action[ListOfLong] = Action.async(validateProto[ListOfLong]) { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- volumeTracingService.find(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) @@ -162,11 +165,12 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer } } - def agglomerateGraphMinCut(annotationId: String, tracingId: String): Action[MinCutParameters] = + def agglomerateGraphMinCut(tracingId: String): Action[MinCutParameters] = Action.async(validateJson[MinCutParameters]) { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- volumeTracingService.find(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) @@ -181,11 +185,12 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer } } - def agglomerateGraphNeighbors(annotationId: String, tracingId: String): Action[NeighborsParameters] = + def agglomerateGraphNeighbors(tracingId: String): Action[NeighborsParameters] = Action.async(validateJson[NeighborsParameters]) { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- volumeTracingService.find(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) @@ -200,10 +205,11 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer } } - def agglomerateSkeleton(annotationId: String, tracingId: String, agglomerateId: Long): Action[AnyContent] = + def agglomerateSkeleton(tracingId: String, agglomerateId: Long): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- volumeTracingService.find(annotationId, tracingId) _ <- bool2Fox(tracing.getHasEditableMapping) ?~> "Cannot query agglomerate skeleton for volume annotation" editableMappingInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala index 79ebfb8138b..f74c54dc999 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala @@ -49,8 +49,7 @@ class SkeletonTracingController @Inject()(val tracingService: SkeletonTracingSer } } - def duplicate(annotationId: String, - tracingId: String, + def duplicate(tracingId: String, version: Option[Long], fromTask: Option[Boolean], editPosition: Option[String], @@ -60,6 +59,7 @@ class SkeletonTracingController @Inject()(val tracingService: SkeletonTracingSer log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId, version, applyUpdates = true) ?~> Messages( "tracing.notFound") editPositionParsed <- Fox.runOptional(editPosition)(Vec3Int.fromUriLiteral) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala index a8c1b04dd60..a7ecc6fb09c 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala @@ -70,11 +70,12 @@ trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends C } } - def get(annotationId: String, tracingId: String, version: Option[Long]): Action[AnyContent] = + def get(tracingId: String, version: Option[Long]): Action[AnyContent] = Action.async { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId, version, applyUpdates = true) ?~> Messages( "tracing.notFound") } yield Ok(tracing.toByteArray).as(protobufMimeType) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 1679838400c..bdf301581d4 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -78,15 +78,13 @@ class VolumeTracingController @Inject()( implicit def unpackMultiple(tracings: VolumeTracings): List[Option[VolumeTracing]] = tracings.tracings.toList.map(_.tracing) - def initialData(annotationId: String, - tracingId: String, - minResolution: Option[Int], - maxResolution: Option[Int]): Action[AnyContent] = + def initialData(tracingId: String, minResolution: Option[Int], maxResolution: Option[Int]): Action[AnyContent] = Action.async { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) initialData <- request.body.asRaw.map(_.asFile) ?~> Messages("zipFile.notFound") tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") resolutionRestrictions = ResolutionRestrictions(minResolution, maxResolution) @@ -119,12 +117,13 @@ class VolumeTracingController @Inject()( } } - def initialDataMultiple(annotationId: String, tracingId: String): Action[AnyContent] = + def initialDataMultiple(tracingId: String): Action[AnyContent] = Action.async { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) initialData <- request.body.asRaw.map(_.asFile) ?~> Messages("zipFile.notFound") tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") resolutions <- tracingService @@ -137,8 +136,7 @@ class VolumeTracingController @Inject()( } } - def allDataZip(annotationId: String, - tracingId: String, + def allDataZip(tracingId: String, volumeDataZipFormat: String, version: Option[Long], voxelSizeFactor: Option[String], @@ -147,6 +145,7 @@ class VolumeTracingController @Inject()( log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId, version) ?~> Messages("tracing.notFound") volumeDataZipFormatParsed <- VolumeDataZipFormat.fromString(volumeDataZipFormat).toFox voxelSizeFactorParsedOpt <- Fox.runOptional(voxelSizeFactor)(Vec3Double.fromUriLiteral) @@ -164,11 +163,12 @@ class VolumeTracingController @Inject()( } } - def data(annotationId: String, tracingId: String): Action[List[WebknossosDataRequest]] = + def data(tracingId: String): Action[List[WebknossosDataRequest]] = Action.async(validateJson[List[WebknossosDataRequest]]) { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") (data, indices) <- if (tracing.getHasEditableMapping) { val mappingLayer = annotationService.editableMappingLayer(annotationId, tracingId, tracing) @@ -185,8 +185,7 @@ class VolumeTracingController @Inject()( private def formatMissingBucketList(indices: List[Int]): String = "[" + indices.mkString(", ") + "]" - def duplicate(annotationId: String, - tracingId: String, + def duplicate(tracingId: String, fromTask: Option[Boolean], minResolution: Option[Int], maxResolution: Option[Int], @@ -198,6 +197,7 @@ class VolumeTracingController @Inject()( logTime(slackNotificationService.noticeSlowRequest) { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") _ = logger.info(s"Duplicating volume tracing $tracingId...") datasetBoundingBox = request.body.asJson.flatMap(_.validateOpt[BoundingBox].asOpt.flatten) @@ -231,11 +231,12 @@ class VolumeTracingController @Inject()( } } - def importVolumeData(annotationId: String, tracingId: String): Action[MultipartFormData[TemporaryFile]] = + def importVolumeData(tracingId: String): Action[MultipartFormData[TemporaryFile]] = Action.async(parse.multipartFormData) { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") currentVersion <- request.body.dataParts("currentVersion").headOption.flatMap(_.toIntOpt).toFox zipFile <- request.body.files.headOption.map(f => new File(f.ref.path.toString)).toFox @@ -249,11 +250,12 @@ class VolumeTracingController @Inject()( } } - def addSegmentIndex(annotationId: String, tracingId: String, dryRun: Boolean): Action[AnyContent] = + def addSegmentIndex(tracingId: String, dryRun: Boolean): Action[AnyContent] = Action.async { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") currentVersion <- annotationService.currentMaterializableVersion(tracingId) before = Instant.now @@ -285,13 +287,14 @@ class VolumeTracingController @Inject()( } } - def requestAdHocMesh(annotationId: String, tracingId: String): Action[WebknossosAdHocMeshRequest] = + def requestAdHocMesh(tracingId: String): Action[WebknossosAdHocMeshRequest] = Action.async(validateJson[WebknossosAdHocMeshRequest]) { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { // The client expects the ad-hoc mesh as a flat float-array. Three consecutive floats form a 3D point, three // consecutive 3D points (i.e., nine floats) form a triangle. // There are no shared vertices between triangles. + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") (vertices: Array[Float], neighbors: List[Int]) <- if (tracing.getHasEditableMapping) { val editableMappingLayer = annotationService.editableMappingLayer(annotationId, tracingId, tracing) @@ -306,10 +309,11 @@ class VolumeTracingController @Inject()( } } - def loadFullMeshStl(annotationId: String, tracingId: String): Action[FullMeshRequest] = + def loadFullMeshStl(tracingId: String): Action[FullMeshRequest] = Action.async(validateJson[FullMeshRequest]) { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) data: Array[Byte] <- fullMeshService.loadFor(annotationId, tracingId, request.body) ?~> "mesh.file.loadChunk.failed" } yield Ok(data) } @@ -321,9 +325,10 @@ class VolumeTracingController @Inject()( private def formatNeighborList(neighbors: List[Int]): String = "[" + neighbors.mkString(", ") + "]" - def findData(annotationId: String, tracingId: String): Action[AnyContent] = Action.async { implicit request => + def findData(tracingId: String): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) positionOpt <- tracingService.findData(annotationId, tracingId) } yield { Ok(Json.obj("position" -> positionOpt, "resolution" -> positionOpt.map(_ => Vec3Int.ones))) @@ -331,10 +336,11 @@ class VolumeTracingController @Inject()( } } - def getSegmentVolume(annotationId: String, tracingId: String): Action[SegmentStatisticsParameters] = + def getSegmentVolume(tracingId: String): Action[SegmentStatisticsParameters] = Action.async(validateJson[SegmentStatisticsParameters]) { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId) mappingName <- annotationService.baseMappingName(annotationId, tracingId, tracing) segmentVolumes <- Fox.serialCombined(request.body.segmentIds) { segmentId => @@ -349,10 +355,11 @@ class VolumeTracingController @Inject()( } } - def getSegmentBoundingBox(annotationId: String, tracingId: String): Action[SegmentStatisticsParameters] = + def getSegmentBoundingBox(tracingId: String): Action[SegmentStatisticsParameters] = Action.async(validateJson[SegmentStatisticsParameters]) { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId) mappingName <- annotationService.baseMappingName(annotationId, tracingId, tracing) segmentBoundingBoxes: List[BoundingBox] <- Fox.serialCombined(request.body.segmentIds) { segmentId => @@ -367,10 +374,11 @@ class VolumeTracingController @Inject()( } } - def getSegmentIndex(annotationId: String, tracingId: String, segmentId: Long): Action[GetSegmentIndexParameters] = + def getSegmentIndex(tracingId: String, segmentId: Long): Action[GetSegmentIndexParameters] = Action.async(validateJson[GetSegmentIndexParameters]) { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) fallbackLayer <- tracingService.getFallbackLayer(annotationId, tracingId) tracing <- tracingService.find(annotationId, tracingId) mappingName <- annotationService.baseMappingName(annotationId, tracingId, tracing) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala index 66ff9cad10e..7108a8104f6 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala @@ -59,10 +59,11 @@ class VolumeTracingZarrStreamingController @Inject()( override def defaultErrorCode: Int = NOT_FOUND - def volumeTracingFolderContent(annotationId: String, tracingId: String, zarrVersion: Int): Action[AnyContent] = + def volumeTracingFolderContent(tracingId: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) additionalFiles = if (zarrVersion == 2) @@ -78,10 +79,11 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def volumeTracingFolderContentJson(annotationId: String, tracingId: String, zarrVersion: Int): Action[AnyContent] = + def volumeTracingFolderContentJson(tracingId: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto(_).toMagLiteral(allowScalar = true)) additionalFiles = if (zarrVersion == 2) @@ -91,15 +93,12 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def volumeTracingMagFolderContent(annotationId: String, - tracingId: String, - mag: String, - zarrVersion: Int): Action[AnyContent] = + def volumeTracingMagFolderContent(tracingId: String, mag: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND - existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND @@ -114,13 +113,11 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def volumeTracingMagFolderContentJson(annotationId: String, - tracingId: String, - mag: String, - zarrVersion: Int): Action[AnyContent] = + def volumeTracingMagFolderContentJson(tracingId: String, mag: String, zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND @@ -130,10 +127,11 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def zArray(annotationId: String, tracingId: String, mag: String): Action[AnyContent] = + def zArray(tracingId: String, mag: String): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND @@ -164,10 +162,11 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def zarrJsonForMag(annotationId: String, tracingId: String, mag: String): Action[AnyContent] = + def zarrJsonForMag(tracingId: String, mag: String): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) @@ -210,7 +209,7 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def zGroup(annotationId: String, tracingId: String): Action[AnyContent] = Action.async { implicit request => + def zGroup(tracingId: String): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { Future(Ok(Json.toJson(NgffGroupHeader(zarr_format = 2)))) } @@ -222,13 +221,12 @@ class VolumeTracingZarrStreamingController @Inject()( * Used by zarr-streaming. */ def zAttrs( - annotationId: String, tracingId: String, ): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND - existingMags = tracing.resolutions.map(vec3IntFromProto) dataSource <- remoteWebknossosClient.getDataSourceForTracing(tracingId) ~> NOT_FOUND omeNgffHeader = NgffMetadata.fromNameVoxelSizeAndMags(tracingId, @@ -239,13 +237,12 @@ class VolumeTracingZarrStreamingController @Inject()( } def zarrJson( - annotationId: String, tracingId: String, ): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND - existingMags = tracing.resolutions.map(vec3IntFromProto) dataSource <- remoteWebknossosClient.getDataSourceForTracing(tracingId) ~> NOT_FOUND omeNgffHeader = NgffMetadataV0_5.fromNameVoxelSizeAndMags(tracingId, @@ -257,15 +254,12 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def zarrSource(annotationId: String, - tracingId: String, - tracingName: Option[String], - zarrVersion: Int): Action[AnyContent] = + def zarrSource(tracingId: String, tracingName: Option[String], zarrVersion: Int): Action[AnyContent] = Action.async { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND - zarrLayer = ZarrSegmentationLayer( name = tracingName.getOrElse(tracingId), largestSegmentId = tracing.largestSegmentId, @@ -280,11 +274,12 @@ class VolumeTracingZarrStreamingController @Inject()( } } - def rawZarrCube(annotationId: String, tracingId: String, mag: String, coordinates: String): Action[AnyContent] = + def rawZarrCube(tracingId: String, mag: String, coordinates: String): Action[AnyContent] = Action.async { implicit request => { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala index ae23b35d8d4..01e87fa3a37 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala @@ -2,7 +2,8 @@ package com.scalableminds.webknossos.tracingstore.tracings import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.tools.{Fox, FoxImplicits, JsonHelper} -import com.scalableminds.webknossos.tracingstore.TracingStoreRedisStore +import com.scalableminds.webknossos.datastore.services.RemoteWebknossosClient +import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingStoreRedisStore} import com.scalableminds.webknossos.tracingstore.annotation.{TSAnnotationService, UpdateActionGroup} import com.scalableminds.webknossos.tracingstore.tracings.TracingType.TracingType import com.scalableminds.webknossos.tracingstore.tracings.volume.MergedVolumeStats @@ -38,6 +39,8 @@ trait TracingService[T <: GeneratedMessage] def temporaryTracingIdStore: TracingStoreRedisStore + def remoteWebknossosClient: TSRemoteWebknossosClient + def tracingMigrationService: TracingMigrationService[T] def annotationService: TSAnnotationService @@ -109,8 +112,6 @@ trait TracingService[T <: GeneratedMessage] } */ - def applyPendingUpdates(tracing: T, tracingId: String, targetVersion: Option[Long]): Fox[T] = Fox.successful(tracing) - def find(annotationId: String, tracingId: String, version: Option[Long] = None, @@ -121,8 +122,11 @@ trait TracingService[T <: GeneratedMessage] implicit tc: TokenContext): Fox[List[Option[T]]] = Fox.combined { selectors.map { - case Some(selector) => // TODO TracingSelector needs annotationIds too - find("dummyAnnotationid", selector.tracingId, selector.version, useCache, applyUpdates).map(Some(_)) + case Some(selector) => + for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(selector.tracingId) + tracing <- find(annotationId, selector.tracingId, selector.version, useCache, applyUpdates).map(Some(_)) + } yield tracing case None => Fox.successful(None) } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala index e952eb33f6f..66c8ac9c9d9 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala @@ -8,7 +8,8 @@ import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.geometry.NamedBoundingBoxProto import com.scalableminds.webknossos.datastore.helpers.{ProtoGeometryImplicits, SkeletonTracingDefaults} import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis -import com.scalableminds.webknossos.tracingstore.TracingStoreRedisStore +import com.scalableminds.webknossos.datastore.services.RemoteWebknossosClient +import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingStoreRedisStore} import com.scalableminds.webknossos.tracingstore.annotation.TSAnnotationService import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.webknossos.tracingstore.tracings.volume.MergedVolumeStats @@ -22,6 +23,7 @@ class SkeletonTracingService @Inject()( val temporaryTracingStore: TemporaryTracingStore[SkeletonTracing], val handledGroupIdStore: TracingStoreRedisStore, val temporaryTracingIdStore: TracingStoreRedisStore, + val remoteWebknossosClient: TSRemoteWebknossosClient, val uncommittedUpdatesStore: TracingStoreRedisStore, val annotationService: TSAnnotationService, val tracingMigrationService: SkeletonTracingMigrationService)(implicit val ec: ExecutionContext) diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 8d074e46e96..6a4af8dd09d 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -1,79 +1,78 @@ -# Routes -# This file defines all application routes (Higher priority routes first) +# Defines tracingstore routes (Higher priority routes first) # ~~~~ # Health endpoint -GET /health @com.scalableminds.webknossos.tracingstore.controllers.Application.health +GET /health @com.scalableminds.webknossos.tracingstore.controllers.Application.health # Annotations (concerns AnnotationProto, not annotation info as stored in postgres) -POST /annotation/save @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.save(annotationId: String) -GET /annotation/:annotationId @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.get(annotationId: String, version: Option[Long]) -POST /annotation/:annotationId/update @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.update(annotationId: String) -GET /annotation/:annotationId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionLog(annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]) -GET /annotation/:annotationId/updateActionStatistics @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionStatistics(annotationId: String) -GET /annotation/:annotationId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.newestVersion(annotationId: String) +POST /annotation/save @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.save(annotationId: String) +GET /annotation/:annotationId @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.get(annotationId: String, version: Option[Long]) +POST /annotation/:annotationId/update @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.update(annotationId: String) +GET /annotation/:annotationId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionLog(annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]) +GET /annotation/:annotationId/updateActionStatistics @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionStatistics(annotationId: String) +GET /annotation/:annotationId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.newestVersion(annotationId: String) # Volume tracings -POST /volume/save @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.save() -POST /volume/:annotationId/:tracingId/initialData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.initialData(annotationId: String, tracingId: String, minResolution: Option[Int], maxResolution: Option[Int]) -POST /volume/:annotationId/:tracingId/initialDataMultiple @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.initialDataMultiple(annotationId: String, tracingId: String) -GET /volume/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.get(annotationId: String, tracingId: String, version: Option[Long]) -GET /volume/:annotationId/:tracingId/allDataZip @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.allDataZip(annotationId: String, tracingId: String, volumeDataZipFormat: String, version: Option[Long], voxelSize: Option[String], voxelSizeUnit: Option[String]) -POST /volume/:annotationId/:tracingId/data @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.data(annotationId: String, tracingId: String) -POST /volume/:annotationId/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.duplicate(annotationId: String, tracingId: String, fromTask: Option[Boolean], minResolution: Option[Int], maxResolution: Option[Int], downsample: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) -POST /volume/:annotationId/:tracingId/adHocMesh @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.requestAdHocMesh(annotationId: String, tracingId: String) -POST /volume/:annotationId/:tracingId/fullMesh.stl @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.loadFullMeshStl(annotationId: String, tracingId: String) -POST /volume/:annotationId/:tracingId/segmentIndex/:segmentId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentIndex(annotationId: String, tracingId: String, segmentId: Long) -POST /volume/:annotationId/:tracingId/importVolumeData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.importVolumeData(annotationId: String, tracingId: String) -POST /volume/:annotationId/:tracingId/addSegmentIndex @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.addSegmentIndex(annotationId: String, tracingId: String, dryRun: Boolean) -GET /volume/:annotationId/:tracingId/findData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.findData(annotationId: String, tracingId: String) -POST /volume/:annotationId/:tracingId/segmentStatistics/volume @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentVolume(annotationId: String, tracingId: String) -POST /volume/:annotationId/:tracingId/segmentStatistics/boundingBox @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentBoundingBox(annotationId: String, tracingId: String) -POST /volume/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getMultiple -POST /volume/mergedFromIds @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromIds(persist: Boolean) -POST /volume/mergedFromContents @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromContents(persist: Boolean) +POST /volume/save @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.save() +POST /volume/:tracingId/initialData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.initialData(tracingId: String, minResolution: Option[Int], maxResolution: Option[Int]) +POST /volume/:tracingId/initialDataMultiple @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.initialDataMultiple(tracingId: String) +GET /volume/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.get(tracingId: String, version: Option[Long]) +GET /volume/:tracingId/allDataZip @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.allDataZip(tracingId: String, volumeDataZipFormat: String, version: Option[Long], voxelSize: Option[String], voxelSizeUnit: Option[String]) +POST /volume/:tracingId/data @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.data(tracingId: String) +POST /volume/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.duplicate(tracingId: String, fromTask: Option[Boolean], minResolution: Option[Int], maxResolution: Option[Int], downsample: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) +POST /volume/:tracingId/adHocMesh @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.requestAdHocMesh(tracingId: String) +POST /volume/:tracingId/fullMesh.stl @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.loadFullMeshStl(tracingId: String) +POST /volume/:tracingId/segmentIndex/:segmentId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentIndex(tracingId: String, segmentId: Long) +POST /volume/:tracingId/importVolumeData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.importVolumeData(tracingId: String) +POST /volume/:tracingId/addSegmentIndex @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.addSegmentIndex(tracingId: String, dryRun: Boolean) +GET /volume/:tracingId/findData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.findData(tracingId: String) +POST /volume/:tracingId/segmentStatistics/volume @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentVolume(tracingId: String) +POST /volume/:tracingId/segmentStatistics/boundingBox @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentBoundingBox(tracingId: String) +POST /volume/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getMultiple +POST /volume/mergedFromIds @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromIds(persist: Boolean) +POST /volume/mergedFromContents @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromContents(persist: Boolean) # Editable Mappings -POST /mapping/:annotationId/:tracingId/makeMappingEditable @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.makeMappingEditable(annotationId: String, tracingId: String) -GET /mapping/:annotationId/:tracingId/info @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.editableMappingInfo(annotationId: String, tracingId: String, version: Option[Long]) -GET /mapping/:tracingId/segmentsForAgglomerate @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.segmentIdsForAgglomerate(tracingId: String, agglomerateId: Long) -POST /mapping/:annotationId/:tracingId/agglomeratesForSegments @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateIdsForSegments(annotationId: String, tracingId: String) -POST /mapping/:annotationId/:tracingId/agglomerateGraphMinCut @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateGraphMinCut(annotationId: String, tracingId: String) -POST /mapping/:annotationId/:tracingId/agglomerateGraphNeighbors @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateGraphNeighbors(annotationId: String, tracingId: String) +POST /mapping/:tracingId/makeMappingEditable @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.makeMappingEditable(tracingId: String) +GET /mapping/:tracingId/info @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.editableMappingInfo(tracingId: String, version: Option[Long]) +GET /mapping/:tracingId/segmentsForAgglomerate @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.segmentIdsForAgglomerate(tracingId: String, agglomerateId: Long) +POST /mapping/:tracingId/agglomeratesForSegments @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateIdsForSegments(tracingId: String) +POST /mapping/:tracingId/agglomerateGraphMinCut @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateGraphMinCut(tracingId: String) +POST /mapping/:tracingId/agglomerateGraphNeighbors @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateGraphNeighbors(tracingId: String) # TODO rename -GET /volume/:annotationId/:tracingId/agglomerateSkeleton/:agglomerateId @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateSkeleton(annotationId: String, tracingId: String, agglomerateId: Long) +GET /volume/:tracingId/agglomerateSkeleton/:agglomerateId @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateSkeleton(tracingId: String, agglomerateId: Long) # Zarr endpoints for volume annotations # Zarr version 2 -GET /volume/zarr/json/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(annotationId: String, tracingId: String, zarrVersion: Int = 2) -GET /volume/zarr/json/:annotationId/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 2) -GET /volume/zarr/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(annotationId: String, tracingId: String, zarrVersion: Int = 2) -GET /volume/zarr/:annotationId/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(annotationId: String, tracingId: String, zarrVersion: Int = 2) -GET /volume/zarr/:annotationId/:tracingId/.zgroup @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zGroup(annotationId: String, tracingId: String) -GET /volume/zarr/:annotationId/:tracingId/.zattrs @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zAttrs(annotationId: String, tracingId: String) -GET /volume/zarr/:annotationId/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(annotationId: String, tracingId: String, tracingName: Option[String], zarrVersion: Int = 2) -GET /volume/zarr/:annotationId/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 2) -GET /volume/zarr/:annotationId/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 2) -GET /volume/zarr/:annotationId/:tracingId/:mag/.zarray @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zArray(annotationId: String, tracingId: String, mag: String) -GET /volume/zarr/:annotationId/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(annotationId: String, tracingId: String, mag: String, coordinates: String) +GET /volume/zarr/json/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(tracingId: String, zarrVersion: Int = 2) +GET /volume/zarr/json/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(tracingId: String, mag: String, zarrVersion: Int = 2) +GET /volume/zarr/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(tracingId: String, zarrVersion: Int = 2) +GET /volume/zarr/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(tracingId: String, zarrVersion: Int = 2) +GET /volume/zarr/:tracingId/.zgroup @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zGroup(tracingId: String) +GET /volume/zarr/:tracingId/.zattrs @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zAttrs(tracingId: String) +GET /volume/zarr/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(tracingId: String, tracingName: Option[String], zarrVersion: Int = 2) +GET /volume/zarr/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(tracingId: String, mag: String, zarrVersion: Int = 2) +GET /volume/zarr/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(tracingId: String, mag: String, zarrVersion: Int = 2) +GET /volume/zarr/:tracingId/:mag/.zarray @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zArray(tracingId: String, mag: String) +GET /volume/zarr/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(tracingId: String, mag: String, coordinates: String) # Zarr version 3 -GET /volume/zarr3_experimental/json/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(annotationId: String, tracingId: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/json/:annotationId/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(annotationId: String, tracingId: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:annotationId/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(annotationId: String, tracingId: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:annotationId/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(annotationId: String, tracingId: String, tracingName: Option[String], zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:annotationId/:tracingId/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJson(annotationId: String, tracingId: String) -GET /volume/zarr3_experimental/:annotationId/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:annotationId/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(annotationId: String, tracingId: String, mag: String, zarrVersion: Int = 3) -GET /volume/zarr3_experimental/:annotationId/:tracingId/:mag/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJsonForMag(annotationId: String, tracingId: String, mag: String) -GET /volume/zarr3_experimental/:annotationId/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(annotationId: String, tracingId: String, mag: String, coordinates: String) +GET /volume/zarr3_experimental/json/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContentJson(tracingId: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/json/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContentJson(tracingId: String, mag: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(tracingId: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingFolderContent(tracingId: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId/zarrSource @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrSource(tracingId: String, tracingName: Option[String], zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJson(tracingId: String) +GET /volume/zarr3_experimental/:tracingId/:mag @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(tracingId: String, mag: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId/:mag/ @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.volumeTracingMagFolderContent(tracingId: String, mag: String, zarrVersion: Int = 3) +GET /volume/zarr3_experimental/:tracingId/:mag/zarr.json @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.zarrJsonForMag(tracingId: String, mag: String) +GET /volume/zarr3_experimental/:tracingId/:mag/:coordinates @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingZarrStreamingController.rawZarrCube(tracingId: String, mag: String, coordinates: String) # Skeleton tracings -POST /skeleton/save @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.save() -POST /skeleton/saveMultiple @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.saveMultiple() -POST /skeleton/mergedFromContents @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.mergedFromContents(persist: Boolean) -POST /skeleton/mergedFromIds @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.mergedFromIds(persist: Boolean) -GET /skeleton/:annotationId/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.get(annotationId: String, tracingId: String, version: Option[Long]) -POST /skeleton/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.getMultiple -POST /skeleton/:annotationId/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.duplicate(annotationId: String, tracingId: String, version: Option[Long], fromTask: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) +POST /skeleton/save @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.save() +POST /skeleton/saveMultiple @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.saveMultiple() +POST /skeleton/mergedFromContents @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.mergedFromContents(persist: Boolean) +POST /skeleton/mergedFromIds @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.mergedFromIds(persist: Boolean) +GET /skeleton/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.get(tracingId: String, version: Option[Long]) +POST /skeleton/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.getMultiple +POST /skeleton/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.duplicate(tracingId: String, version: Option[Long], fromTask: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) From 1ff9b9102861d1b14c0ab83e94810bf96a28f586 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 26 Sep 2024 14:01:39 +0200 Subject: [PATCH 084/150] resolve some todos, close annotationUpdates grpc handle, remove unused functions --- .../WKRemoteTracingStoreClient.scala | 2 +- frontend/javascripts/admin/admin_rest_api.ts | 2 +- .../AnnotationTransactionService.scala | 11 +++++------ .../annotation/TSAnnotationService.scala | 4 ++-- .../controllers/VolumeTracingController.scala | 12 ------------ .../tracings/TracingDataStore.scala | 10 +--------- .../tracings/TracingSelector.scala | 2 +- .../EditableMappingService.scala | 7 +------ .../tracings/volume/VolumeTracingService.scala | 18 ------------------ ...calableminds.webknossos.tracingstore.routes | 3 +-- 10 files changed, 13 insertions(+), 58 deletions(-) diff --git a/app/models/annotation/WKRemoteTracingStoreClient.scala b/app/models/annotation/WKRemoteTracingStoreClient.scala index d6bcf96bcdd..590d5862cc3 100644 --- a/app/models/annotation/WKRemoteTracingStoreClient.scala +++ b/app/models/annotation/WKRemoteTracingStoreClient.scala @@ -87,7 +87,7 @@ class WKRemoteTracingStoreClient( rpc(s"${tracingStore.url}/tracings/annotation/save") .addQueryString("token" -> RpcTokenHolder.webknossosToken) .addQueryString("annotationId" -> annotationId.toString) - .postProto[AnnotationProto](annotationProto) // TODO why didn’t the failure bubble up? + .postProto[AnnotationProto](annotationProto) } def duplicateSkeletonTracing(skeletonTracingId: String, diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts index 89d6c5fe1cf..28204b26da5 100644 --- a/frontend/javascripts/admin/admin_rest_api.ts +++ b/frontend/javascripts/admin/admin_rest_api.ts @@ -2155,7 +2155,7 @@ export function getEditableAgglomerateSkeleton( ): Promise { return doWithToken((token) => Request.receiveArraybuffer( - `${tracingStoreUrl}/tracings/volume/${tracingId}/agglomerateSkeleton/${agglomerateId}?token=${token}`, + `${tracingStoreUrl}/tracings/mapping/${tracingId}/agglomerateSkeleton/${agglomerateId}?token=${token}`, // The webworker code cannot do proper error handling and always expects an array buffer from the server. // However, the server might send an error json instead of an array buffer. Therefore, don't use the webworker code. { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala index 63e700c8bcf..0bedbdab47a 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala @@ -18,12 +18,11 @@ import javax.inject.Inject import scala.concurrent.ExecutionContext import scala.concurrent.duration._ -class AnnotationTransactionService @Inject()( - handledGroupIdStore: TracingStoreRedisStore, // TODO: instantiate here rather than with injection, give fix namespace prefix? - uncommittedUpdatesStore: TracingStoreRedisStore, - volumeTracingService: VolumeTracingService, - tracingDataStore: TracingDataStore, - annotationService: TSAnnotationService) +class AnnotationTransactionService @Inject()(handledGroupIdStore: TracingStoreRedisStore, + uncommittedUpdatesStore: TracingStoreRedisStore, + volumeTracingService: VolumeTracingService, + tracingDataStore: TracingDataStore, + annotationService: TSAnnotationService) extends KeyValueStoreImplicits with LazyLogging { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 6f3a995bcb8..10f8d15a8d3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -216,7 +216,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss targetVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext) = { val volumeWithEditableMapping = annotationWithTracings.volumesThatHaveEditableMapping logger.info(s"fetching editable mappings ${volumeWithEditableMapping.map(_._2).mkString(",")}") - // TODO intersect with editable mapping updates? + // TODO perf optimization: intersect with editable mapping updates? unless requested for { idInfoUpdaterTuples <- Fox.serialCombined(volumeWithEditableMapping) { case (volumeTracing, volumeTracingId) => @@ -373,7 +373,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss def updateActionStatistics(tracingId: String): Fox[JsObject] = for { - updateActionGroups <- tracingDataStore.skeletonUpdates.getMultipleVersions(tracingId)( + updateActionGroups <- tracingDataStore.annotationUpdates.getMultipleVersions(tracingId)( fromJsonBytes[List[UpdateAction]]) updateActions = updateActionGroups.flatten } yield { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index bdf301581d4..243761dea06 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -275,18 +275,6 @@ class VolumeTracingController @Inject()( } } - def updateActionLog(tracingId: String, - newestVersion: Option[Long] = None, - oldestVersion: Option[Long] = None): Action[AnyContent] = Action.async { implicit request => - log() { - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { - for { - updateLog <- tracingService.updateActionLog(tracingId, newestVersion, oldestVersion) - } yield Ok(updateLog) - } - } - } - def requestAdHocMesh(tracingId: String): Action[WebknossosAdHocMeshRequest] = Action.async(validateJson[WebknossosAdHocMeshRequest]) { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala index a836eb43a7c..e5e5d0ad777 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala @@ -22,16 +22,12 @@ class TracingDataStore @Inject()(config: TracingStoreConfig, lazy val skeletons = new FossilDBClient("skeletons", config, slackNotificationService) - lazy val skeletonUpdates = new FossilDBClient("skeletonUpdates", config, slackNotificationService) - lazy val volumes = new FossilDBClient("volumes", config, slackNotificationService) lazy val volumeData = new FossilDBClient("volumeData", config, slackNotificationService) lazy val volumeSegmentIndex = new FossilDBClient("volumeSegmentIndex", config, slackNotificationService) - lazy val volumeUpdates = new FossilDBClient("volumeUpdates", config, slackNotificationService) - lazy val editableMappingsInfo = new FossilDBClient("editableMappingsInfo", config, slackNotificationService) lazy val editableMappingsAgglomerateToGraph = @@ -40,8 +36,6 @@ class TracingDataStore @Inject()(config: TracingStoreConfig, lazy val editableMappingsSegmentToAgglomerate = new FossilDBClient("editableMappingsSegmentToAgglomerate", config, slackNotificationService) - lazy val editableMappingUpdates = new FossilDBClient("editableMappingUpdates", config, slackNotificationService) - lazy val annotations = new FossilDBClient("annotations", config, slackNotificationService) lazy val annotationUpdates = new FossilDBClient("annotationUpdates", config, slackNotificationService) @@ -49,14 +43,12 @@ class TracingDataStore @Inject()(config: TracingStoreConfig, private def shutdown(): Unit = { healthClient.shutdown() skeletons.shutdown() - skeletonUpdates.shutdown() + annotationUpdates.shutdown() volumes.shutdown() volumeData.shutdown() - volumeUpdates.shutdown() editableMappingsInfo.shutdown() editableMappingsAgglomerateToGraph.shutdown() editableMappingsSegmentToAgglomerate.shutdown() - editableMappingUpdates.shutdown() volumeSegmentIndex.shutdown() () } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingSelector.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingSelector.scala index 14598c9d5aa..0329c57e34a 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingSelector.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingSelector.scala @@ -2,6 +2,6 @@ package com.scalableminds.webknossos.tracingstore.tracings import play.api.libs.json.{Json, OFormat} -case class TracingSelector(tracingId: String, version: Option[Long] = None) // TODO must pass annotation id +case class TracingSelector(tracingId: String, version: Option[Long] = None) object TracingSelector { implicit val jsonFormat: OFormat[TracingSelector] = Json.format[TracingSelector] } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index b8ea620e1b3..e5299e4b3e1 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -22,7 +22,6 @@ import com.scalableminds.webknossos.datastore.services.{ AdHocMeshServiceHolder, BinaryDataService } -import com.scalableminds.webknossos.tracingstore.annotation.{UpdateAction, UpdateActionGroup} import com.scalableminds.webknossos.tracingstore.tracings.volume.ReversionHelper import com.scalableminds.webknossos.tracingstore.tracings.{ FallbackDataHelper, @@ -155,7 +154,6 @@ class EditableMappingService @Inject()( } } yield () - */ private def duplicateSegmentToAgglomerate(sourceTracingId: String, newId: String, newVersion: Long): Fox[Unit] = { val iterator = @@ -190,14 +188,11 @@ class EditableMappingService @Inject()( }.toList) } yield () } + */ def assertTracingHasEditableMapping(tracing: VolumeTracing)(implicit ec: ExecutionContext): Fox[Unit] = bool2Fox(tracing.getHasEditableMapping) ?~> "annotation.volume.noEditableMapping" - def getBaseMappingName(tracingId: String): Fox[Option[String]] = - // TODO - Fox.successful(None) - def findSegmentIdAtPositionIfNeeded(remoteFallbackLayer: RemoteFallbackLayer, positionOpt: Option[Vec3Int], segmentIdOpt: Option[Long], diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 31b40df0961..b56e5c8787e 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -613,24 +613,6 @@ class VolumeTracingService @Inject()( additionalAxes = AdditionalAxis.fromProtosAsOpt(tracing.additionalAxes) ) - def updateActionLog(tracingId: String, - newestVersion: Option[Long] = None, - oldestVersion: Option[Long] = None): Fox[JsValue] = { - def versionedTupleToJson(tuple: (Long, List[CompactVolumeUpdateAction])): JsObject = - Json.obj( - "version" -> tuple._1, - "value" -> Json.toJson(tuple._2) - ) - - for { - volumeTracings <- tracingDataStore.volumeUpdates.getMultipleVersionsAsVersionValueTuple( - tracingId, - newestVersion, - oldestVersion)(fromJsonBytes[List[CompactVolumeUpdateAction]]) - updateActionGroupsJs = volumeTracings.map(versionedTupleToJson) - } yield Json.toJson(updateActionGroupsJs) - } - def updateResolutionList(tracingId: String, tracing: VolumeTracing, resolutions: Set[Vec3Int], diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 6a4af8dd09d..7d9cd93f228 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -39,8 +39,7 @@ GET /mapping/:tracingId/segmentsForAgglomerate POST /mapping/:tracingId/agglomeratesForSegments @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateIdsForSegments(tracingId: String) POST /mapping/:tracingId/agglomerateGraphMinCut @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateGraphMinCut(tracingId: String) POST /mapping/:tracingId/agglomerateGraphNeighbors @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateGraphNeighbors(tracingId: String) -# TODO rename -GET /volume/:tracingId/agglomerateSkeleton/:agglomerateId @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateSkeleton(tracingId: String, agglomerateId: Long) +GET /mapping/:tracingId/agglomerateSkeleton/:agglomerateId @com.scalableminds.webknossos.tracingstore.controllers.EditableMappingController.agglomerateSkeleton(tracingId: String, agglomerateId: Long) # Zarr endpoints for volume annotations # Zarr version 2 From a7ee147a60b650422ab3fe99b8c57fd7ce7dd6e9 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 26 Sep 2024 14:32:48 +0200 Subject: [PATCH 085/150] batching for update action log --- .../annotation/TSAnnotationService.scala | 27 ++++++++++++++----- .../controllers/TSAnnotationController.scala | 3 ++- .../EditableMappingService.scala | 9 +------ 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 10f8d15a8d3..072712bb0d9 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -124,20 +124,25 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss Fox.failure("not implemented") // TODO create tracing object (ask wk for needed parameters e.g. fallback layer info?) - def updateActionLog(annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]): Fox[JsValue] = { + def updateActionLog(annotationId: String, newestVersion: Long, oldestVersion: Long)( + implicit ec: ExecutionContext): Fox[JsValue] = { def versionedTupleToJson(tuple: (Long, List[UpdateAction])): JsObject = Json.obj( "version" -> tuple._1, "value" -> Json.toJson(tuple._2) ) + val batchRanges = batchRangeInclusive(oldestVersion, newestVersion, batchSize = 100) for { - updateActionGroups <- tracingDataStore.annotationUpdates.getMultipleVersionsAsVersionValueTuple( - annotationId, - newestVersion, - oldestVersion)(fromJsonBytes[List[UpdateAction]]) - updateActionGroupsJs = updateActionGroups.map(versionedTupleToJson) - } yield Json.toJson(updateActionGroupsJs) + updateActionBatches <- Fox.serialCombined(batchRanges.toList) { batchRange => + val batchFrom = batchRange._1 + val batchTo = batchRange._2 + tracingDataStore.annotationUpdates.getMultipleVersionsAsVersionValueTuple( + annotationId, + Some(batchTo), + Some(batchFrom))(fromJsonBytes[List[UpdateAction]]) + } + } yield Json.toJson(updateActionBatches.flatten.map(versionedTupleToJson)) } def get(annotationId: String, version: Option[Long])(implicit ec: ExecutionContext, @@ -417,4 +422,12 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss editableMappingInfo <- getEditableMappingInfo(annotationId, tracingId) } yield Some(editableMappingInfo.baseMappingName) else Fox.successful(tracing.mappingName) + + private def batchRangeInclusive(from: Long, to: Long, batchSize: Long): Seq[(Long, Long)] = + (0L to ((to - from) / batchSize)).map { batchIndex => + val batchFrom = batchIndex * batchSize + from + val batchTo = Math.min(to, (batchIndex + 1) * batchSize + from - 1) + (batchFrom, batchTo) + } + } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index f4fe393cc77..755a31aee67 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -58,7 +58,8 @@ class TSAnnotationController @Inject()( log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readAnnotation(annotationId)) { for { - updateLog <- annotationService.updateActionLog(annotationId, newestVersion, oldestVersion) + newestMaterializableVersion <- annotationService.currentMaterializableVersion(annotationId) + updateLog <- annotationService.updateActionLog(annotationId, newestVersion.getOrElse(newestMaterializableVersion), oldestVersion.getOrElse(0)) } yield Ok(updateLog) } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index e5299e4b3e1..79ad28415ad 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -573,12 +573,5 @@ class EditableMappingService @Inject()( } } yield () - */ - - private def batchRangeInclusive(from: Long, to: Long, batchSize: Long): Seq[(Long, Long)] = - (0L to ((to - from) / batchSize)).map { batchIndex => - val batchFrom = batchIndex * batchSize + from - val batchTo = Math.min(to, (batchIndex + 1) * batchSize + from - 1) - (batchFrom, batchTo) - } + */ } From 65056ec631f29bbfbc6e33ab477b230fe6b17340 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 26 Sep 2024 14:39:13 +0200 Subject: [PATCH 086/150] remove unused stuff from TracingService --- .../tracings/TracingService.scala | 65 +------------------ .../volume/VolumeTracingService.scala | 2 - 2 files changed, 2 insertions(+), 65 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala index 01e87fa3a37..58cb7da7d16 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala @@ -1,17 +1,14 @@ package com.scalableminds.webknossos.tracingstore.tracings import com.scalableminds.util.accesscontext.TokenContext -import com.scalableminds.util.tools.{Fox, FoxImplicits, JsonHelper} -import com.scalableminds.webknossos.datastore.services.RemoteWebknossosClient +import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingStoreRedisStore} -import com.scalableminds.webknossos.tracingstore.annotation.{TSAnnotationService, UpdateActionGroup} +import com.scalableminds.webknossos.tracingstore.annotation.TSAnnotationService import com.scalableminds.webknossos.tracingstore.tracings.TracingType.TracingType import com.scalableminds.webknossos.tracingstore.tracings.volume.MergedVolumeStats import com.typesafe.scalalogging.LazyLogging -import play.api.http.Status.CONFLICT import net.liftweb.common.Box import play.api.i18n.MessagesProvider -import play.api.libs.json._ import scalapb.{GeneratedMessage, GeneratedMessageCompanion} import java.util.UUID @@ -47,10 +44,6 @@ trait TracingService[T <: GeneratedMessage] def dummyTracing: T - val handledGroupIdStore: TracingStoreRedisStore - - val uncommittedUpdatesStore: TracingStoreRedisStore - implicit def tracingCompanion: GeneratedMessageCompanion[T] // this should be longer than maxCacheTime in webknossos/AnnotationStore @@ -61,46 +54,9 @@ trait TracingService[T <: GeneratedMessage] // to provide useful error messages to the user if the temporary tracing is no longer present private val temporaryIdStoreTimeout = 10 days - private val handledGroupCacheExpiry: FiniteDuration = 24 hours - - private def transactionGroupKey(tracingId: String, transactionId: String, transactionGroupIndex: Int, version: Long) = - s"transactionGroup___${tracingId}___${transactionId}___${transactionGroupIndex}___$version" - protected def temporaryIdKey(tracingId: String) = s"temporaryTracingId___$tracingId" - private def patternFor(tracingId: String, transactionId: String) = - s"transactionGroup___${tracingId}___${transactionId}___*" - - def saveUncommitted(tracingId: String, - transactionId: String, - transactionGroupIndex: Int, - version: Long, - updateGroup: UpdateActionGroup, - expiry: FiniteDuration): Fox[Unit] = - for { - _ <- Fox.runIf(transactionGroupIndex > 0)( - Fox.assertTrue( - uncommittedUpdatesStore.contains(transactionGroupKey( - tracingId, - transactionId, - transactionGroupIndex - 1, - version))) ?~> s"Incorrect transaction index. Got: $transactionGroupIndex but ${transactionGroupIndex - 1} does not exist" ~> CONFLICT) - _ <- uncommittedUpdatesStore.insert(transactionGroupKey(tracingId, transactionId, transactionGroupIndex, version), - Json.toJson(updateGroup).toString(), - Some(expiry)) - } yield () - - def getAllUncommittedFor(tracingId: String, transactionId: String): Fox[List[UpdateActionGroup]] = - for { - raw: Seq[String] <- uncommittedUpdatesStore.findAllConditional(patternFor(tracingId, transactionId)) - parsed: Seq[UpdateActionGroup] = raw.flatMap(itemAsString => - JsonHelper.jsResultToOpt(Json.parse(itemAsString).validate[UpdateActionGroup])) - } yield parsed.toList.sortBy(_.transactionGroupIndex) - - def removeAllUncommittedFor(tracingId: String, transactionId: String): Fox[Unit] = - uncommittedUpdatesStore.removeAllConditional(patternFor(tracingId, transactionId)) - /* // TODO ? add this to migration? private def migrateTracing(tracingFox: Fox[T], tracingId: String): Fox[T] = tracingMigrationService.migrateTracing(tracingFox).flatMap { @@ -144,23 +100,6 @@ trait TracingService[T <: GeneratedMessage] } } - private def handledGroupKey(tracingId: String, transactionId: String, version: Long, transactionGroupIndex: Int) = - s"handledGroup___${tracingId}___${transactionId}___${version}___$transactionGroupIndex" - - def saveToHandledGroupIdStore(tracingId: String, - transactionId: String, - version: Long, - transactionGroupIndex: Int): Fox[Unit] = { - val key = handledGroupKey(tracingId, transactionId, version, transactionGroupIndex) - handledGroupIdStore.insert(key, "()", Some(handledGroupCacheExpiry)) - } - - def handledGroupIdStoreContains(tracingId: String, - transactionId: String, - version: Long, - transactionGroupIndex: Int): Fox[Boolean] = - handledGroupIdStore.contains(handledGroupKey(tracingId, transactionId, version, transactionGroupIndex)) - def merge(tracings: Seq[T], mergedVolumeStats: MergedVolumeStats, newEditableMappingIdOpt: Option[String]): Box[T] def remapTooLargeTreeIds(tracing: T): T = tracing diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index b56e5c8787e..96cfea1f6dc 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -37,8 +37,6 @@ import net.liftweb.common.{Box, Empty, Failure, Full} import play.api.i18n.{Messages, MessagesProvider} import play.api.libs.Files import play.api.libs.Files.TemporaryFileCreator -import play.api.libs.json.{JsObject, JsValue, Json} - import java.io._ import java.nio.file.Paths import java.util.Base64 From 1d9bb89795e5829f9d6eda6b176438d951c26276 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 26 Sep 2024 14:47:40 +0200 Subject: [PATCH 087/150] handle dummy annotation id + dummy tracing id --- app/controllers/UserTokenController.scala | 33 +++++++++++-------- .../WKRemoteTracingStoreController.scala | 11 +++++-- .../TSRemoteWebknossosClient.scala | 2 +- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/app/controllers/UserTokenController.scala b/app/controllers/UserTokenController.scala index 3f95d0ff51f..29580465eeb 100644 --- a/app/controllers/UserTokenController.scala +++ b/app/controllers/UserTokenController.scala @@ -39,7 +39,6 @@ object RpcTokenHolder { class UserTokenController @Inject()(datasetDAO: DatasetDAO, datasetService: DatasetService, - annotationDAO: AnnotationDAO, annotationPrivateLinkDAO: AnnotationPrivateLinkDAO, userService: UserService, organizationDAO: OrganizationDAO, @@ -184,18 +183,24 @@ class UserTokenController @Inject()(datasetDAO: DatasetDAO, case _ => Fox.successful(false) } - // TODO is a dummy annotation id needed? - for { - annotation <- annotationInformationProvider.provideAnnotation(annotationId, userBox)(GlobalAccessContext) ?~> "annotation.notFound" - annotationAccessByToken <- token.map(annotationPrivateLinkDAO.findOneByAccessToken).getOrElse(Fox.empty).futureBox - allowedByToken = annotationAccessByToken.exists(annotation._id == _._annotation) - restrictions <- annotationInformationProvider.restrictionsFor( - AnnotationIdentifier(annotation.typ, annotation._id))(GlobalAccessContext) ?~> "restrictions.notFound" - allowedByUser <- checkRestrictions(restrictions) ?~> "restrictions.failedToCheck" - allowed = allowedByToken || allowedByUser - } yield { - if (allowed) UserAccessAnswer(granted = true) - else UserAccessAnswer(granted = false, Some(s"No ${mode.toString} access to tracing")) + if (annotationId == ObjectId.dummyId.toString) { + Fox.successful(UserAccessAnswer(granted = true)) + } else { + for { + annotation <- annotationInformationProvider.provideAnnotation(annotationId, userBox)(GlobalAccessContext) ?~> "annotation.notFound" + annotationAccessByToken <- token + .map(annotationPrivateLinkDAO.findOneByAccessToken) + .getOrElse(Fox.empty) + .futureBox + allowedByToken = annotationAccessByToken.exists(annotation._id == _._annotation) + restrictions <- annotationInformationProvider.restrictionsFor( + AnnotationIdentifier(annotation.typ, annotation._id))(GlobalAccessContext) ?~> "restrictions.notFound" + allowedByUser <- checkRestrictions(restrictions) ?~> "restrictions.failedToCheck" + allowed = allowedByToken || allowedByUser + } yield { + if (allowed) UserAccessAnswer(granted = true) + else UserAccessAnswer(granted = false, Some(s"No ${mode.toString} access to tracing")) + } } } @@ -208,7 +213,7 @@ class UserTokenController @Inject()(datasetDAO: DatasetDAO, jobBox <- jobDAO.findOne(jobIdValidated)(DBAccessContext(userBox)).futureBox answer = jobBox match { case Full(_) => UserAccessAnswer(granted = true) - case _ => UserAccessAnswer(granted = false, Some(s"No ${mode} access to job export")) + case _ => UserAccessAnswer(granted = false, Some(s"No $mode access to job export")) } } yield answer } diff --git a/app/controllers/WKRemoteTracingStoreController.scala b/app/controllers/WKRemoteTracingStoreController.scala index 1476dba7c28..f3b488b8060 100644 --- a/app/controllers/WKRemoteTracingStoreController.scala +++ b/app/controllers/WKRemoteTracingStoreController.scala @@ -5,6 +5,7 @@ import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId import com.scalableminds.webknossos.tracingstore.TracingUpdatesReport +import com.scalableminds.webknossos.tracingstore.tracings.TracingIds import javax.inject.Inject import models.analytics.{AnalyticsService, UpdateAnnotationEvent, UpdateAnnotationViewOnlyEvent} @@ -118,9 +119,13 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore Action.async { implicit request => tracingStoreService.validateAccess(name, key) { _ => implicit val ctx: DBAccessContext = GlobalAccessContext - for { - annotation <- annotationInformationProvider.annotationForTracing(tracingId) ?~> s"No annotation for tracing $tracingId" - } yield Ok(Json.toJson(annotation._id)) + if (tracingId == TracingIds.dummyTracingId) { + Fox.successful(Ok(Json.toJson(ObjectId.dummyId))) + } else { + for { + annotation <- annotationInformationProvider.annotationForTracing(tracingId) ?~> s"No annotation for tracing $tracingId" + } yield Ok(Json.toJson(annotation._id)) + } } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala index 8024af01d16..7835962dd20 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala @@ -22,7 +22,7 @@ import scala.concurrent.ExecutionContext import scala.concurrent.duration.DurationInt case class TracingUpdatesReport(annotationId: String, - // TODO stats per tracing id? + // TODO stats per tracing id? coordinate with frontend timestamps: List[Instant], statistics: Option[JsObject], significantChangesCount: Int, From d092ece22ad1247a78ef47ca7aaa3642471c4388 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 26 Sep 2024 15:11:28 +0200 Subject: [PATCH 088/150] distinguish between updateUserBoundingBoxVisibility in skeleton + volume tracing --- .../model/sagas/skeletontracing_saga.ts | 4 +- .../oxalis/model/sagas/update_actions.ts | 24 +++++-- .../oxalis/model/sagas/volumetracing_saga.tsx | 4 +- .../javascripts/oxalis/view/version_entry.tsx | 6 +- .../annotation/UpdateActions.scala | 67 ++++++++++--------- 5 files changed, 63 insertions(+), 42 deletions(-) diff --git a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts index 96b98f14640..4c56c6f837c 100644 --- a/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/skeletontracing_saga.ts @@ -27,7 +27,7 @@ import { updateTreeEdgesVisibility, updateNode, updateSkeletonTracing, - updateUserBoundingBoxes, + updateUserBoundingBoxesInSkeletonTracing, updateTree, updateTreeGroups, } from "oxalis/model/sagas/update_actions"; @@ -636,7 +636,7 @@ export function* diffSkeletonTracing( } if (!_.isEqual(prevSkeletonTracing.userBoundingBoxes, skeletonTracing.userBoundingBoxes)) { - yield updateUserBoundingBoxes(skeletonTracing.userBoundingBoxes); + yield updateUserBoundingBoxesInSkeletonTracing(skeletonTracing.userBoundingBoxes); } } export default [ diff --git a/frontend/javascripts/oxalis/model/sagas/update_actions.ts b/frontend/javascripts/oxalis/model/sagas/update_actions.ts index 2d6e15ec4fc..b7ab2a80b87 100644 --- a/frontend/javascripts/oxalis/model/sagas/update_actions.ts +++ b/frontend/javascripts/oxalis/model/sagas/update_actions.ts @@ -34,7 +34,12 @@ export type CreateSegmentUpdateAction = ReturnType; export type DeleteSegmentUpdateAction = ReturnType; export type DeleteSegmentDataUpdateAction = ReturnType; -type UpdateUserBoundingBoxesUpdateAction = ReturnType; +type UpdateUserBoundingBoxesInSkeletonTracingUpdateAction = ReturnType< + typeof updateUserBoundingBoxesInSkeletonTracing +>; +type UpdateUserBoundingBoxesInVolumeTracingUpdateAction = ReturnType< + typeof updateUserBoundingBoxesInVolumeTracing +>; export type UpdateBucketUpdateAction = ReturnType; type UpdateSegmentGroupsUpdateAction = ReturnType; @@ -61,7 +66,8 @@ export type UpdateAction = | DeleteEdgeUpdateAction | UpdateSkeletonTracingUpdateAction | UpdateVolumeTracingUpdateAction - | UpdateUserBoundingBoxesUpdateAction + | UpdateUserBoundingBoxesInSkeletonTracingUpdateAction + | UpdateUserBoundingBoxesInVolumeTracingUpdateAction | CreateSegmentUpdateAction | UpdateSegmentUpdateAction | DeleteSegmentUpdateAction @@ -314,9 +320,19 @@ export function updateVolumeTracing( }, } as const; } -export function updateUserBoundingBoxes(userBoundingBoxes: Array) { +export function updateUserBoundingBoxesInSkeletonTracing( + userBoundingBoxes: Array, +) { + return { + name: "updateUserBoundingBoxesInSkeletonTracing", + value: { + boundingBoxes: convertUserBoundingBoxesFromFrontendToServer(userBoundingBoxes), + }, + } as const; +} +export function updateUserBoundingBoxesInVolumeTracing(userBoundingBoxes: Array) { return { - name: "updateUserBoundingBoxes", + name: "updateUserBoundingBoxesInVolumeTracing", value: { boundingBoxes: convertUserBoundingBoxesFromFrontendToServer(userBoundingBoxes), }, diff --git a/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx b/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx index 42ecceb0d37..938c04d76b7 100644 --- a/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx +++ b/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx @@ -95,7 +95,7 @@ import { deleteSegmentVolumeAction, removeFallbackLayer, updateSegmentVolumeAction, - updateUserBoundingBoxes, + updateUserBoundingBoxesInVolumeTracing, updateVolumeTracing, updateMappingName, } from "oxalis/model/sagas/update_actions"; @@ -700,7 +700,7 @@ export function* diffVolumeTracing( } if (!_.isEqual(prevVolumeTracing.userBoundingBoxes, volumeTracing.userBoundingBoxes)) { - yield updateUserBoundingBoxes(volumeTracing.userBoundingBoxes); + yield updateUserBoundingBoxesInVolumeTracing(volumeTracing.userBoundingBoxes); } if (prevVolumeTracing !== volumeTracing) { diff --git a/frontend/javascripts/oxalis/view/version_entry.tsx b/frontend/javascripts/oxalis/view/version_entry.tsx index 4302438ded2..f9133b26a2f 100644 --- a/frontend/javascripts/oxalis/view/version_entry.tsx +++ b/frontend/javascripts/oxalis/view/version_entry.tsx @@ -65,7 +65,11 @@ const descriptionFns: Record Descr description: "Created the annotation.", icon: , }), - updateUserBoundingBoxes: (): Description => ({ + updateUserBoundingBoxesInSkeletonTracing: (): Description => ({ + description: "Updated a bounding box.", + icon: , + }), + updateUserBoundingBoxesInVolumeTracing: (): Description => ({ description: "Updated a bounding box.", icon: , }), diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala index 8c7d0e34cb8..b54db784dc3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala @@ -66,41 +66,42 @@ object UpdateAction { val jsonValue = (json \ "value").as[JsObject] (json \ "name").as[String] match { // Skeleton - case "createTree" => deserialize[CreateTreeSkeletonAction](jsonValue) - case "deleteTree" => deserialize[DeleteTreeSkeletonAction](jsonValue) - case "updateTree" => deserialize[UpdateTreeSkeletonAction](jsonValue) - case "mergeTree" => deserialize[MergeTreeSkeletonAction](jsonValue) - case "moveTreeComponent" => deserialize[MoveTreeComponentSkeletonAction](jsonValue) - case "createNode" => deserialize[CreateNodeSkeletonAction](jsonValue, shouldTransformPositions = true) - case "deleteNode" => deserialize[DeleteNodeSkeletonAction](jsonValue) - case "updateNode" => deserialize[UpdateNodeSkeletonAction](jsonValue, shouldTransformPositions = true) - case "createEdge" => deserialize[CreateEdgeSkeletonAction](jsonValue) - case "deleteEdge" => deserialize[DeleteEdgeSkeletonAction](jsonValue) - case "updateTreeGroups" => deserialize[UpdateTreeGroupsSkeletonAction](jsonValue) - case "updateSkeletonTracing" => deserialize[UpdateTracingSkeletonAction](jsonValue) - case "updateTreeVisibility" => deserialize[UpdateTreeVisibilitySkeletonAction](jsonValue) - case "updateTreeGroupVisibility" => deserialize[UpdateTreeGroupVisibilitySkeletonAction](jsonValue) - case "updateTreeEdgesVisibility" => deserialize[UpdateTreeEdgesVisibilitySkeletonAction](jsonValue) - case "updateUserBoundingBoxes" => deserialize[UpdateUserBoundingBoxesSkeletonAction](jsonValue) - case "updateUserBoundingBoxVisibility" => + case "createTree" => deserialize[CreateTreeSkeletonAction](jsonValue) + case "deleteTree" => deserialize[DeleteTreeSkeletonAction](jsonValue) + case "updateTree" => deserialize[UpdateTreeSkeletonAction](jsonValue) + case "mergeTree" => deserialize[MergeTreeSkeletonAction](jsonValue) + case "moveTreeComponent" => deserialize[MoveTreeComponentSkeletonAction](jsonValue) + case "createNode" => deserialize[CreateNodeSkeletonAction](jsonValue, shouldTransformPositions = true) + case "deleteNode" => deserialize[DeleteNodeSkeletonAction](jsonValue) + case "updateNode" => deserialize[UpdateNodeSkeletonAction](jsonValue, shouldTransformPositions = true) + case "createEdge" => deserialize[CreateEdgeSkeletonAction](jsonValue) + case "deleteEdge" => deserialize[DeleteEdgeSkeletonAction](jsonValue) + case "updateTreeGroups" => deserialize[UpdateTreeGroupsSkeletonAction](jsonValue) + case "updateSkeletonTracing" => deserialize[UpdateTracingSkeletonAction](jsonValue) + case "updateTreeVisibility" => deserialize[UpdateTreeVisibilitySkeletonAction](jsonValue) + case "updateTreeGroupVisibility" => deserialize[UpdateTreeGroupVisibilitySkeletonAction](jsonValue) + case "updateTreeEdgesVisibility" => deserialize[UpdateTreeEdgesVisibilitySkeletonAction](jsonValue) + case "updateUserBoundingBoxesInSkeletonTracing" => deserialize[UpdateUserBoundingBoxesSkeletonAction](jsonValue) + case "updateUserBoundingBoxVisibilityInSkeletonTracing" => deserialize[UpdateUserBoundingBoxVisibilitySkeletonAction](jsonValue) // Volume case "updateBucket" => deserialize[UpdateBucketVolumeAction](jsonValue) case "updateVolumeTracing" => deserialize[UpdateTracingVolumeAction](jsonValue) - case "updateUserBoundingBoxes" => + case "updateUserBoundingBoxesInVolumeTracing" => deserialize[UpdateUserBoundingBoxesVolumeAction](jsonValue) // TODO: rename key (must be different from skeleton action) - case "updateUserBoundingBoxVisibility" => deserialize[UpdateUserBoundingBoxVisibilityVolumeAction](jsonValue) - case "removeFallbackLayer" => deserialize[RemoveFallbackLayerVolumeAction](jsonValue) - case "importVolumeTracing" => deserialize[ImportVolumeDataVolumeAction](jsonValue) - case "updateTdCameraSkeleton" => deserialize[UpdateTdCameraSkeletonAction](jsonValue) // TODO deduplicate? - case "updateTdCameraVolume" => deserialize[UpdateTdCameraVolumeAction](jsonValue) - case "createSegment" => deserialize[CreateSegmentVolumeAction](jsonValue) - case "updateSegment" => deserialize[UpdateSegmentVolumeAction](jsonValue) - case "updateSegmentGroups" => deserialize[UpdateSegmentGroupsVolumeAction](jsonValue) - case "deleteSegment" => deserialize[DeleteSegmentVolumeAction](jsonValue) - case "deleteSegmentData" => deserialize[DeleteSegmentDataVolumeAction](jsonValue) - case "updateMappingName" => deserialize[UpdateMappingNameVolumeAction](jsonValue) + case "updateUserBoundingBoxVisibilityInVolumeTracing" => + deserialize[UpdateUserBoundingBoxVisibilityVolumeAction](jsonValue) + case "removeFallbackLayer" => deserialize[RemoveFallbackLayerVolumeAction](jsonValue) + case "importVolumeTracing" => deserialize[ImportVolumeDataVolumeAction](jsonValue) + case "updateTdCameraSkeleton" => deserialize[UpdateTdCameraSkeletonAction](jsonValue) // TODO deduplicate? + case "updateTdCameraVolume" => deserialize[UpdateTdCameraVolumeAction](jsonValue) + case "createSegment" => deserialize[CreateSegmentVolumeAction](jsonValue) + case "updateSegment" => deserialize[UpdateSegmentVolumeAction](jsonValue) + case "updateSegmentGroups" => deserialize[UpdateSegmentGroupsVolumeAction](jsonValue) + case "deleteSegment" => deserialize[DeleteSegmentVolumeAction](jsonValue) + case "deleteSegmentData" => deserialize[DeleteSegmentDataVolumeAction](jsonValue) + case "updateMappingName" => deserialize[UpdateMappingNameVolumeAction](jsonValue) // Editable Mapping case "mergeAgglomerate" => deserialize[MergeAgglomerateUpdateAction](jsonValue) @@ -164,10 +165,10 @@ object UpdateAction { Json.obj("name" -> "updateTreeEdgesVisibility", "value" -> Json.toJson(s)(UpdateTreeEdgesVisibilitySkeletonAction.jsonFormat)) case s: UpdateUserBoundingBoxesSkeletonAction => - Json.obj("name" -> "updateUserBoundingBoxes", + Json.obj("name" -> "updateUserBoundingBoxesInSkeletonTracing", "value" -> Json.toJson(s)(UpdateUserBoundingBoxesSkeletonAction.jsonFormat)) case s: UpdateUserBoundingBoxVisibilitySkeletonAction => - Json.obj("name" -> "updateUserBoundingBoxVisibility", + Json.obj("name" -> "updateUserBoundingBoxVisibilityInSkeletonTracing", "value" -> Json.toJson(s)(UpdateUserBoundingBoxVisibilitySkeletonAction.jsonFormat)) case s: UpdateTdCameraSkeletonAction => Json.obj("name" -> "updateTdCameraSkeleton", "value" -> Json.toJson(s)(UpdateTdCameraSkeletonAction.jsonFormat)) @@ -178,10 +179,10 @@ object UpdateAction { case s: UpdateTracingVolumeAction => Json.obj("name" -> "updateVolumeTracing", "value" -> Json.toJson(s)(UpdateTracingVolumeAction.jsonFormat)) case s: UpdateUserBoundingBoxesVolumeAction => - Json.obj("name" -> "updateUserBoundingBoxes", + Json.obj("name" -> "updateUserBoundingBoxesInVolumeTracing", "value" -> Json.toJson(s)(UpdateUserBoundingBoxesVolumeAction.jsonFormat)) case s: UpdateUserBoundingBoxVisibilityVolumeAction => - Json.obj("name" -> "updateUserBoundingBoxVisibility", + Json.obj("name" -> "updateUserBoundingBoxVisibilityInVolumeTracing", "value" -> Json.toJson(s)(UpdateUserBoundingBoxVisibilityVolumeAction.jsonFormat)) case s: RemoveFallbackLayerVolumeAction => Json.obj("name" -> "removeFallbackLayer", "value" -> Json.toJson(s)(RemoveFallbackLayerVolumeAction.jsonFormat)) From 2bec9e3c13f819fd1e31e2b8dcc7b9e7ace5f518 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 26 Sep 2024 16:09:26 +0200 Subject: [PATCH 089/150] wip: prepare revertToVersion workflow --- .../annotation/TSAnnotationService.scala | 22 +++++++++++++++---- .../EditableMappingController.scala | 3 +-- .../EditableMappingService.scala | 13 +++++------ .../EditableMappingUpdater.scala | 2 +- .../skeleton/SkeletonTracingService.scala | 1 - 5 files changed, 25 insertions(+), 16 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 072712bb0d9..755bd9f62a8 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -78,11 +78,12 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss if (desiredVersion == existingVersion) Fox.successful(List()) else { for { - updateActionGroups <- tracingDataStore.annotationUpdates.getMultipleVersions( + updateActionGroupsWithVersions <- tracingDataStore.annotationUpdates.getMultipleVersionsAsVersionValueTuple( annotationId, Some(desiredVersion), Some(existingVersion + 1))(fromJsonBytes[List[UpdateAction]]) - } yield updateActionGroups.reverse.flatten + updateActionGroupsWithVersionsIroned = ironOutReversionFolds(updateActionGroupsWithVersions) + } yield updateActionGroupsWithVersionsIroned } // TODO option to dry apply? @@ -113,12 +114,22 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss annotationWithTracings.applyVolumeAction(a) case a: EditableMappingUpdateAction => annotationWithTracings.applyEditableMappingAction(a) + case a: RevertToVersionUpdateAction => + revertToVersion(annotationId, annotationWithTracings, a) case _: BucketMutatingVolumeUpdateAction => Fox.successful(annotationWithTracings) // No-op, as bucket-mutating actions are performed eagerly, so not here. case _ => Fox.failure(s"Received unsupported AnnotationUpdateAction action ${Json.toJson(updateAction)}") } } yield updated + private def revertToVersion( + annotationId: String, + annotationWithTracings: AnnotationWithTracings, + revertAction: RevertToVersionUpdateAction)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = + // Note: works only after “ironing out” the update action groups + // TODO: read old annotationProto, tracing, buckets, segment indeces + Fox.successful(annotationWithTracings) + def createTracing(a: AddLayerAnnotationUpdateAction)( implicit ec: ExecutionContext): Fox[Either[SkeletonTracing, VolumeTracing]] = Fox.failure("not implemented") @@ -315,8 +326,6 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss case Full(annotationWithTracings) => remainingUpdates match { case List() => Fox.successful(annotationWithTracings) - case RevertToVersionUpdateAction(sourceVersion, _, _, _) :: tail => - ??? case update :: tail => updateIter(applyUpdate(annotationId, annotationWithTracings, update, targetVersion), tail) } @@ -335,6 +344,11 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } } + private def ironOutReversionFolds( + updateActionGroupsWithVersions: List[(Long, List[UpdateAction])]): List[UpdateAction] = + // TODO: if the source version is in the current update list, it needs to be ironed out. in case of overlaps, iron out from the back. + updateActionGroupsWithVersions.flatMap(_._2) + private def flushUpdatedTracings(annotationWithTracings: AnnotationWithTracings)(implicit ec: ExecutionContext) = // TODO skip some flushes to save disk space (e.g. skeletons only nth version, or only if requested?) for { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala index cc781fe939d..96430c6e33f 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala @@ -128,9 +128,8 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- volumeTracingService.find(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) - remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) agglomerateGraphBox: Box[AgglomerateGraph] <- editableMappingService - .getAgglomerateGraphForId(tracingId, tracing.version, agglomerateId, remoteFallbackLayer) + .getAgglomerateGraphForId(tracingId, tracing.version, agglomerateId) .futureBox segmentIds <- agglomerateGraphBox match { case Full(agglomerateGraph) => Fox.successful(agglomerateGraph.segments) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index 79ad28415ad..dcc8c012cc6 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -107,8 +107,8 @@ class EditableMappingService @Inject()( adHocMeshServiceHolder.tracingStoreAdHocMeshConfig = (binaryDataService, 30 seconds, 1) private val adHocMeshService: AdHocMeshService = adHocMeshServiceHolder.tracingStoreAdHocMeshService - // TODO - private lazy val materializedInfoCache: AlfuCache[(String, Long), EditableMappingInfo] = AlfuCache(maxCapacity = 100) + // TODO cache materialized stuff again, for e.g. faster bucket loading + // private lazy val materializedInfoCache: AlfuCache[(String, Long), EditableMappingInfo] = AlfuCache(maxCapacity = 100) private lazy val segmentToAgglomerateChunkCache: AlfuCache[(String, Long, Long), Seq[(Long, Long)]] = AlfuCache() @@ -298,7 +298,7 @@ class EditableMappingService @Inject()( remoteFallbackLayer: RemoteFallbackLayer, agglomerateId: Long)(implicit tc: TokenContext): Fox[Array[Byte]] = for { - agglomerateGraphBox <- getAgglomerateGraphForId(tracingId, version, agglomerateId, remoteFallbackLayer).futureBox + agglomerateGraphBox <- getAgglomerateGraphForId(tracingId, version, agglomerateId).futureBox skeletonBytes <- agglomerateGraphBox match { case Full(agglomerateGraph) => Fox.successful(agglomerateGraphToSkeleton(tracingId, agglomerateGraph, remoteFallbackLayer, agglomerateId)) @@ -405,10 +405,7 @@ class EditableMappingService @Inject()( adHocMeshService.requestAdHocMeshViaActor(adHocMeshRequest) } - def getAgglomerateGraphForId(tracingId: String, - version: Long, - agglomerateId: Long, - remoteFallbackLayer: RemoteFallbackLayer): Fox[AgglomerateGraph] = + def getAgglomerateGraphForId(tracingId: String, version: Long, agglomerateId: Long): Fox[AgglomerateGraph] = for { agglomerateGraph <- agglomerateToGraphCache.getOrLoad( (tracingId, agglomerateId, version), @@ -429,7 +426,7 @@ class EditableMappingService @Inject()( agglomerateId: Long, remoteFallbackLayer: RemoteFallbackLayer)(implicit tc: TokenContext): Fox[AgglomerateGraph] = for { - agglomerateGraphBox <- getAgglomerateGraphForId(tracingId, version, agglomerateId, remoteFallbackLayer).futureBox + agglomerateGraphBox <- getAgglomerateGraphForId(tracingId, version, agglomerateId).futureBox agglomerateGraph <- agglomerateGraphBox match { case Full(agglomerateGraph) => Fox.successful(agglomerateGraph) case Empty => diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index 5dc1e3017ba..715a3055984 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -455,7 +455,7 @@ class EditableMappingUpdater( for { agglomerateId <- agglomerateIdFromAgglomerateGraphKey(graphKey) _ <- editableMappingService - .getAgglomerateGraphForId(tracingId, revertAction.sourceVersion, agglomerateId, remoteFallbackLayer) + .getAgglomerateGraphForId(tracingId, revertAction.sourceVersion, agglomerateId) .futureBox .map { case Full(graphData) => agglomerateToGraphBuffer.put(graphKey, (graphData, false)) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala index 66c8ac9c9d9..1ee7bb354c1 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala @@ -8,7 +8,6 @@ import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.geometry.NamedBoundingBoxProto import com.scalableminds.webknossos.datastore.helpers.{ProtoGeometryImplicits, SkeletonTracingDefaults} import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis -import com.scalableminds.webknossos.datastore.services.RemoteWebknossosClient import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingStoreRedisStore} import com.scalableminds.webknossos.tracingstore.annotation.TSAnnotationService import com.scalableminds.webknossos.tracingstore.tracings._ From 538395f53d986cd8ae6511e9a54dad4652e44e78 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 7 Oct 2024 11:46:34 +0200 Subject: [PATCH 090/150] wip revert to version --- .../annotation/AnnotationReversion.scala | 15 ++++ .../annotation/TSAnnotationService.scala | 89 ++++++++++++------- .../skeleton/SkeletonTracingService.scala | 6 +- .../volume/VolumeTracingService.scala | 6 +- 4 files changed, 81 insertions(+), 35 deletions(-) create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala new file mode 100644 index 00000000000..4d4bba4b08b --- /dev/null +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala @@ -0,0 +1,15 @@ +package com.scalableminds.webknossos.tracingstore.annotation + +import com.scalableminds.util.tools.Fox + +import scala.concurrent.ExecutionContext + +trait AnnotationReversion { + + def revertDistributedElements(annotationId: String, + annotationWithTracings: AnnotationWithTracings, + revertAction: RevertToVersionUpdateAction)(implicit ec: ExecutionContext): Fox[Unit] = + // TODO segment index, volume buckets, proofreading data + Fox.successful(()) + +} diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 755bd9f62a8..b58965d35b8 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -4,7 +4,7 @@ import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox import com.scalableminds.util.tools.Fox.{box2Fox, option2Fox} -import com.scalableminds.webknossos.datastore.Annotation.AnnotationProto +import com.scalableminds.webknossos.datastore.Annotation.{AnnotationLayerTypeProto, AnnotationProto} import com.scalableminds.webknossos.datastore.EditableMappingInfo.EditableMappingInfo import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing @@ -52,6 +52,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss extends KeyValueStoreImplicits with FallbackDataHelper with ProtoGeometryImplicits + with AnnotationReversion with LazyLogging { def reportUpdates(annotationId: String, updateGroups: List[UpdateActionGroup])(implicit tc: TokenContext): Fox[Unit] = @@ -122,13 +123,21 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } } yield updated - private def revertToVersion( - annotationId: String, - annotationWithTracings: AnnotationWithTracings, - revertAction: RevertToVersionUpdateAction)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = + private def revertToVersion(annotationId: String, + annotationWithTracings: AnnotationWithTracings, + revertAction: RevertToVersionUpdateAction)( + implicit ec: ExecutionContext, + tc: TokenContext): Fox[AnnotationWithTracings] = // Note: works only after “ironing out” the update action groups // TODO: read old annotationProto, tracing, buckets, segment indeces - Fox.successful(annotationWithTracings) + for { + sourceAnnotation <- getWithTracings(annotationId, + Some(revertAction.sourceVersion), + List.empty, + List.empty, + requestAll = true) // TODO do we need to request the others? + _ <- revertDistributedElements(annotationId, sourceAnnotation, revertAction) + } yield sourceAnnotation def createTracing(a: AddLayerAnnotationUpdateAction)( implicit ec: ExecutionContext): Fox[Either[SkeletonTracing, VolumeTracing]] = @@ -159,14 +168,15 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss def get(annotationId: String, version: Option[Long])(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationProto] = for { - withTracings <- getWithTracings(annotationId, version, List.empty, List.empty) + withTracings <- getWithTracings(annotationId, version, List.empty, List.empty, requestAll = false) } yield withTracings.annotation - def getWithTracings(annotationId: String, - version: Option[Long], - requestedSkeletonTracingIds: List[String], - requestedVolumeTracingIds: List[String])(implicit ec: ExecutionContext, - tc: TokenContext): Fox[AnnotationWithTracings] = + def getWithTracings( + annotationId: String, + version: Option[Long], + requestedSkeletonTracingIds: List[String], + requestedVolumeTracingIds: List[String], + requestAll: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = for { annotationWithVersion <- tracingDataStore.annotations.get(annotationId, version)(fromProtoBytes[AnnotationProto]) ?~> "getAnnotation.failed" annotation = annotationWithVersion.value @@ -174,14 +184,15 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss annotationId, version, requestedSkeletonTracingIds, - requestedVolumeTracingIds) ?~> "applyUpdates.failed" + requestedVolumeTracingIds, + requestAll) ?~> "applyUpdates.failed" } yield updated def getEditableMappingInfo(annotationId: String, tracingId: String, version: Option[Long] = None)( implicit ec: ExecutionContext, tc: TokenContext): Fox[EditableMappingInfo] = for { - annotation <- getWithTracings(annotationId, version, List.empty, List(tracingId)) ?~> "getWithTracings.failed" + annotation <- getWithTracings(annotationId, version, List.empty, List(tracingId), requestAll = false) ?~> "getWithTracings.failed" tracing <- annotation.getEditableMappingInfo(tracingId) ?~> "getEditableMapping.failed" } yield tracing @@ -202,20 +213,21 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss targetVersion) } yield annotationWithTracings.addEditableMapping(action.actionTracingId, editableMappingInfo.value, updater) - private def applyPendingUpdates(annotation: AnnotationProto, - annotationId: String, - targetVersionOpt: Option[Long], - requestedSkeletonTracingIds: List[String], - requestedVolumeTracingIds: List[String])( - implicit ec: ExecutionContext, - tc: TokenContext): Fox[AnnotationWithTracings] = + private def applyPendingUpdates( + annotation: AnnotationProto, + annotationId: String, + targetVersionOpt: Option[Long], + requestedSkeletonTracingIds: List[String], + requestedVolumeTracingIds: List[String], + requestAll: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = for { targetVersion <- determineTargetVersion(annotation, annotationId, targetVersionOpt) ?~> "determineTargetVersion.failed" updates <- findPendingUpdates(annotationId, annotation.version, targetVersion) ?~> "findPendingUpdates.failed" annotationWithTracings <- findTracingsForUpdates(annotation, updates, requestedSkeletonTracingIds, - requestedVolumeTracingIds) ?~> "findTracingsForUpdates.failed" + requestedVolumeTracingIds, + requestAll) ?~> "findTracingsForUpdates.failed" annotationWithTracingsAndMappings <- findEditableMappingsForUpdates(annotationId, annotationWithTracings, updates, @@ -283,23 +295,34 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss annotation: AnnotationProto, updates: List[UpdateAction], requestedSkeletonTracingIds: List[String], - requestedVolumeTracingIds: List[String])(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = { - val skeletonTracingIds = (updates.flatMap { - case u: SkeletonUpdateAction => Some(u.actionTracingId) - case _ => None - } ++ requestedSkeletonTracingIds).distinct - val volumeTracingIds = (updates.flatMap { - case u: VolumeUpdateAction => Some(u.actionTracingId) - case _ => None - } ++ requestedVolumeTracingIds).distinct + requestedVolumeTracingIds: List[String], + requestAll: Boolean)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = { + val skeletonTracingIds = + if (requestAll) + annotation.layers.filter(_.`type` == AnnotationLayerTypeProto.skeleton).map(_.tracingId) + else { + (updates.flatMap { + case u: SkeletonUpdateAction => Some(u.actionTracingId) + case _ => None + } ++ requestedSkeletonTracingIds).distinct + } + val volumeTracingIds = + if (requestAll) + annotation.layers.filter(_.`type` == AnnotationLayerTypeProto.volume).map(_.tracingId) + else { + (updates.flatMap { + case u: VolumeUpdateAction => Some(u.actionTracingId) + case _ => None + } ++ requestedVolumeTracingIds).distinct + } logger.info(s"fetching volumes $volumeTracingIds and skeletons $skeletonTracingIds") for { - skeletonTracings <- Fox.serialCombined(skeletonTracingIds)( + skeletonTracings <- Fox.serialCombined(skeletonTracingIds.toList)( id => tracingDataStore.skeletons.get[SkeletonTracing](id, Some(annotation.version), mayBeEmpty = Some(true))( fromProtoBytes[SkeletonTracing])) - volumeTracings <- Fox.serialCombined(volumeTracingIds)( + volumeTracings <- Fox.serialCombined(volumeTracingIds.toList)( id => tracingDataStore.volumes .get[VolumeTracing](id, Some(annotation.version), mayBeEmpty = Some(true))(fromProtoBytes[VolumeTracing])) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala index 1ee7bb354c1..0512226823f 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala @@ -46,7 +46,11 @@ class SkeletonTracingService @Inject()( Fox.successful(dummyTracing) else { for { - annotation <- annotationService.getWithTracings(annotationId, version, List(tracingId), List.empty) // TODO is applyUpdates still needed? + annotation <- annotationService.getWithTracings(annotationId, + version, + List(tracingId), + List.empty, + requestAll = false) // TODO is applyUpdates still needed? tracing <- annotation.getSkeleton(tracingId) } yield tracing } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 96cfea1f6dc..bf8b16b1bb3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -185,7 +185,11 @@ class VolumeTracingService @Inject()( Fox.successful(dummyTracing) else { for { - annotation <- annotationService.getWithTracings(annotationId, version, List.empty, List(tracingId)) // TODO is applyUpdates still needed? + annotation <- annotationService.getWithTracings(annotationId, + version, + List.empty, + List(tracingId), + requestAll = false) // TODO is applyUpdates still needed? tracing <- annotation.getVolume(tracingId) } yield tracing } From 8b8b4c0d298c51d0e149dfec227403aa140cd09c Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 7 Oct 2024 13:27:39 +0200 Subject: [PATCH 091/150] revert to version format --- .../annotation/TSAnnotationService.scala | 2 ++ .../annotation/UpdateActions.scala | 6 +++--- .../updating/SkeletonUpdateActions.scala | 21 ------------------- 3 files changed, 5 insertions(+), 24 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index b58965d35b8..476aa3f5490 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -136,6 +136,8 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss List.empty, List.empty, requestAll = true) // TODO do we need to request the others? + _ = logger.info( + s"reverting to suorceVersion ${revertAction.sourceVersion}. got sourceAnnotation with version ${sourceAnnotation.version}") _ <- revertDistributedElements(annotationId, sourceAnnotation, revertAction) } yield sourceAnnotation diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala index b54db784dc3..6b56c292440 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala @@ -13,7 +13,6 @@ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{ DeleteTreeSkeletonAction, MergeTreeSkeletonAction, MoveTreeComponentSkeletonAction, - RevertToVersionSkeletonAction, UpdateNodeSkeletonAction, UpdateTdCameraSkeletonAction, UpdateTracingSkeletonAction, @@ -112,6 +111,7 @@ object UpdateAction { case "deleteLayerFromAnnotation" => deserialize[DeleteLayerAnnotationUpdateAction](jsonValue) case "updateLayerMetadata" => deserialize[UpdateLayerMetadataAnnotationUpdateAction](jsonValue) case "updateMetadataOfAnnotation" => deserialize[UpdateMetadataAnnotationUpdateAction](jsonValue) + case "revertToVersion" => deserialize[RevertToVersionUpdateAction](jsonValue) case unknownAction: String => JsError(s"Invalid update action s'$unknownAction'") } @@ -153,8 +153,6 @@ object UpdateAction { Json.obj("name" -> "updateTreeGroups", "value" -> Json.toJson(s)(UpdateTreeGroupsSkeletonAction.jsonFormat)) case s: UpdateTracingSkeletonAction => Json.obj("name" -> "updateSkeletonTracing", "value" -> Json.toJson(s)(UpdateTracingSkeletonAction.jsonFormat)) - case s: RevertToVersionSkeletonAction => - Json.obj("name" -> "revertToVersion", "value" -> Json.toJson(s)(RevertToVersionSkeletonAction.jsonFormat)) case s: UpdateTreeVisibilitySkeletonAction => Json.obj("name" -> "updateTreeVisibility", "value" -> Json.toJson(s)(UpdateTreeVisibilitySkeletonAction.jsonFormat)) @@ -220,6 +218,8 @@ object UpdateAction { case s: UpdateMetadataAnnotationUpdateAction => Json.obj("name" -> "updateMetadataOfAnnotation", "value" -> Json.toJson(s)(UpdateMetadataAnnotationUpdateAction.jsonFormat)) + case s: RevertToVersionUpdateAction => + Json.obj("name" -> "revertToVersion", "value" -> Json.toJson(s)(RevertToVersionUpdateAction.jsonFormat)) } } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala index aa82e9af23b..41f3988feb2 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala @@ -404,24 +404,6 @@ case class UpdateTracingSkeletonAction(activeNode: Option[Int], this.copy(actionAuthorId = authorId) } -case class RevertToVersionSkeletonAction(sourceVersion: Long, - actionTracingId: String, - actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) - extends SkeletonUpdateAction { - override def applyOn(tracing: SkeletonTracing): SkeletonTracing = - throw new Exception("RevertToVersionAction applied on unversioned tracing") - - override def addTimestamp(timestamp: Long): UpdateAction = - this.copy(actionTimestamp = Some(timestamp)) - - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - - override def addAuthorId(authorId: Option[String]): UpdateAction = - this.copy(actionAuthorId = authorId) -} - case class UpdateTreeVisibilitySkeletonAction(treeId: Int, isVisible: Boolean, actionTracingId: String, @@ -607,9 +589,6 @@ object UpdateTreeGroupsSkeletonAction { object UpdateTracingSkeletonAction { implicit val jsonFormat: OFormat[UpdateTracingSkeletonAction] = Json.format[UpdateTracingSkeletonAction] } -object RevertToVersionSkeletonAction { - implicit val jsonFormat: OFormat[RevertToVersionSkeletonAction] = Json.format[RevertToVersionSkeletonAction] -} object UpdateTreeVisibilitySkeletonAction { implicit val jsonFormat: OFormat[UpdateTreeVisibilitySkeletonAction] = Json.format[UpdateTreeVisibilitySkeletonAction] } From 0abf0789d47f6a4977b8e8250cac6f7daa39b37d Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 7 Oct 2024 14:21:19 +0200 Subject: [PATCH 092/150] fix update action order --- .../annotation/AnnotationWithTracings.scala | 7 ++++++- .../annotation/TSAnnotationService.scala | 18 +++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index 37b17dfb8f5..42c41169f3d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -13,6 +13,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ } import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.SkeletonUpdateAction import com.scalableminds.webknossos.tracingstore.tracings.volume.ApplyableVolumeUpdateAction +import com.typesafe.scalalogging.LazyLogging import net.liftweb.common.{Box, Failure, Full} import scala.concurrent.ExecutionContext @@ -20,7 +21,8 @@ import scala.concurrent.ExecutionContext case class AnnotationWithTracings( annotation: AnnotationProto, tracingsById: Map[String, Either[SkeletonTracing, VolumeTracing]], - editableMappingsByTracingId: Map[String, (EditableMappingInfo, EditableMappingUpdater)]) { + editableMappingsByTracingId: Map[String, (EditableMappingInfo, EditableMappingUpdater)]) + extends LazyLogging { def getSkeleton(tracingId: String): Box[SkeletonTracing] = for { @@ -142,4 +144,7 @@ case class AnnotationWithTracings( } yield () } + def skeletonStats: String = + f"skeleton with ${getSkeletons.map(_._2).map(_.trees.map(_.nodes.length).sum).mkString} nodes" + } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 476aa3f5490..6a3207058c4 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -131,13 +131,14 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss // Note: works only after “ironing out” the update action groups // TODO: read old annotationProto, tracing, buckets, segment indeces for { - sourceAnnotation <- getWithTracings(annotationId, - Some(revertAction.sourceVersion), - List.empty, - List.empty, - requestAll = true) // TODO do we need to request the others? + sourceAnnotation: AnnotationWithTracings <- getWithTracings( + annotationId, + Some(revertAction.sourceVersion), + List.empty, + List.empty, + requestAll = true) // TODO do we need to request the others? _ = logger.info( - s"reverting to suorceVersion ${revertAction.sourceVersion}. got sourceAnnotation with version ${sourceAnnotation.version}") + s"reverting to suorceVersion ${revertAction.sourceVersion}. got sourceAnnotation with version ${sourceAnnotation.version} with ${sourceAnnotation.skeletonStats}") _ <- revertDistributedElements(annotationId, sourceAnnotation, revertAction) } yield sourceAnnotation @@ -352,6 +353,8 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss remainingUpdates match { case List() => Fox.successful(annotationWithTracings) case update :: tail => + logger.info( + f"${remainingUpdates.length} remainingUpdates, current skeleton ${annotationWithTracings.skeletonStats})") updateIter(applyUpdate(annotationId, annotationWithTracings, update, targetVersion), tail) } case _ => annotationWithTracingsFox @@ -362,6 +365,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss for { updated <- updateIter(Some(annotation), updates) updatedWithNewVerson = updated.withVersion(targetVersion) + _ = logger.info(s"flushing, with ${updated.skeletonStats}") _ <- updatedWithNewVerson.flushBufferedUpdates() _ <- flushUpdatedTracings(updatedWithNewVerson) _ <- flushAnnotationInfo(annotationId, updatedWithNewVerson) @@ -372,7 +376,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss private def ironOutReversionFolds( updateActionGroupsWithVersions: List[(Long, List[UpdateAction])]): List[UpdateAction] = // TODO: if the source version is in the current update list, it needs to be ironed out. in case of overlaps, iron out from the back. - updateActionGroupsWithVersions.flatMap(_._2) + updateActionGroupsWithVersions.reverse.flatMap(_._2) private def flushUpdatedTracings(annotationWithTracings: AnnotationWithTracings)(implicit ec: ExecutionContext) = // TODO skip some flushes to save disk space (e.g. skeletons only nth version, or only if requested?) From 1127a9217779abaf72cc978dae810f9441eaac1e Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 8 Oct 2024 10:15:45 +0200 Subject: [PATCH 093/150] resolve circular dependency between the services --- .../TSRemoteWebknossosClient.scala | 3 +- .../annotation/AnnotationReversion.scala | 17 +- .../AnnotationTransactionService.scala | 25 ++- .../annotation/TSAnnotationService.scala | 69 ++++++- .../EditableMappingController.scala | 16 +- .../SkeletonTracingController.scala | 128 +++++++++++-- .../controllers/TracingController.scala | 131 -------------- .../controllers/VolumeTracingController.scala | 170 ++++++++++++++---- ...VolumeTracingZarrStreamingController.scala | 20 +-- .../tracings/TracingService.scala | 22 --- .../skeleton/SkeletonTracingService.scala | 20 --- .../tracings/volume/TSFullMeshService.scala | 4 +- .../VolumeSegmentStatisticsService.scala | 8 +- .../volume/VolumeTracingDownsampling.scala | 33 ++-- .../volume/VolumeTracingService.scala | 74 +++----- 15 files changed, 410 insertions(+), 330 deletions(-) delete mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala index 7835962dd20..70852b533a6 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala @@ -54,10 +54,11 @@ class TSRemoteWebknossosClient @Inject()( .silent .post(Json.toJson(tracingUpdatesReport)) - def getDataSourceForTracing(tracingId: String): Fox[DataSourceLike] = + def getDataSourceForTracing(tracingId: String)(implicit tc: TokenContext): Fox[DataSourceLike] = rpc(s"$webknossosUri/api/tracingstores/$tracingStoreName/dataSource") .addQueryString("tracingId" -> tracingId) .addQueryString("key" -> tracingStoreKey) + .withTokenFromContext .getWithJsonResponse[DataSourceLike] def getDataStoreUriForDataSource(organizationId: String, datasetName: String): Fox[String] = diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala index 4d4bba4b08b..c26f5bf7bfc 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala @@ -1,15 +1,26 @@ package com.scalableminds.webknossos.tracingstore.annotation +import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.tools.Fox +import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing +import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeSegmentIndexBuffer import scala.concurrent.ExecutionContext trait AnnotationReversion { def revertDistributedElements(annotationId: String, - annotationWithTracings: AnnotationWithTracings, - revertAction: RevertToVersionUpdateAction)(implicit ec: ExecutionContext): Fox[Unit] = + currentAnnotationWithTracings: AnnotationWithTracings, + sourceAnnotationWithTracings: AnnotationWithTracings, + revertAction: RevertToVersionUpdateAction, + newVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Unit] = // TODO segment index, volume buckets, proofreading data - Fox.successful(()) + for { + _ <- Fox.serialCombined(sourceAnnotationWithTracings.getVolumes) { + // Only volume data for volume layers present in the *source annotation* needs to be reverted. + case (tracingId, sourceTracing) => Fox.successful(()) + //revertVolumeData(annotationId, tracingId, sourceTracing, revertAction.sourceVersion, newVersion) + } + } yield () } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala index 0bedbdab47a..e96ffd31641 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala @@ -41,12 +41,12 @@ class AnnotationTransactionService @Inject()(handledGroupIdStore: TracingStoreRe private def patternFor(annotationId: String, transactionId: String) = s"transactionGroup___${annotationId}___${transactionId}___*" - def saveUncommitted(annotationId: String, - transactionId: String, - transactionGroupIndex: Int, - version: Long, - updateGroup: UpdateActionGroup, - expiry: FiniteDuration)(implicit ec: ExecutionContext): Fox[Unit] = + private def saveUncommitted(annotationId: String, + transactionId: String, + transactionGroupIndex: Int, + version: Long, + updateGroup: UpdateActionGroup, + expiry: FiniteDuration)(implicit ec: ExecutionContext): Fox[Unit] = for { _ <- Fox.runIf(transactionGroupIndex > 0)( Fox.assertTrue( @@ -188,8 +188,17 @@ class AnnotationTransactionService @Inject()(handledGroupIdStore: TracingStoreRe updateActionsJson <- Fox.successful(Json.toJson(preprocessActionsForStorage(updateActionGroup))) _ <- tracingDataStore.annotationUpdates.put(annotationId, updateActionGroup.version, updateActionsJson) bucketMutatingActions = findBucketMutatingActions(updateActionGroup) - _ <- Fox.runIf(bucketMutatingActions.nonEmpty)( - volumeTracingService.applyBucketMutatingActions(annotationId, bucketMutatingActions, updateActionGroup.version)) + actionsGrouped: Map[String, List[BucketMutatingVolumeUpdateAction]] = bucketMutatingActions.groupBy( + _.actionTracingId) + _ <- Fox.serialCombined(actionsGrouped.keys.toList) { volumeTracingId => + for { + tracing <- annotationService.findVolume(annotationId, volumeTracingId) + _ <- volumeTracingService.applyBucketMutatingActions(volumeTracingId, + tracing, + bucketMutatingActions, + updateActionGroup.version) + } yield () + } } yield () private def findBucketMutatingActions(updateActionGroup: UpdateActionGroup): List[BucketMutatingVolumeUpdateAction] = diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 6a3207058c4..567e9d47c0a 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -15,6 +15,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ EditableMappingUpdateAction, EditableMappingUpdater } +import com.scalableminds.webknossos.tracingstore.tracings.skeleton.SkeletonTracingService import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{ CreateNodeSkeletonAction, DeleteNodeSkeletonAction, @@ -25,12 +26,15 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ ApplyableVolumeUpdateAction, BucketMutatingVolumeUpdateAction, UpdateMappingNameVolumeAction, + VolumeTracingService, VolumeUpdateAction } import com.scalableminds.webknossos.tracingstore.tracings.{ FallbackDataHelper, KeyValueStoreImplicits, TracingDataStore, + TracingIds, + TracingSelector, VersionedKeyValuePair } import com.scalableminds.webknossos.tracingstore.{ @@ -47,6 +51,8 @@ import scala.concurrent.ExecutionContext class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknossosClient, editableMappingService: EditableMappingService, + volumeTracingService: VolumeTracingService, + skeletonTracingService: SkeletonTracingService, val remoteDatastoreClient: TSRemoteDatastoreClient, tracingDataStore: TracingDataStore) extends KeyValueStoreImplicits @@ -139,7 +145,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss requestAll = true) // TODO do we need to request the others? _ = logger.info( s"reverting to suorceVersion ${revertAction.sourceVersion}. got sourceAnnotation with version ${sourceAnnotation.version} with ${sourceAnnotation.skeletonStats}") - _ <- revertDistributedElements(annotationId, sourceAnnotation, revertAction) + // _ <- revertDistributedElements(annotationId, annotationWithTracings, sourceAnnotation, revertAction) } yield sourceAnnotation def createTracing(a: AddLayerAnnotationUpdateAction)( @@ -473,4 +479,65 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss (batchFrom, batchTo) } + def findVolume(annotationId: String, + tracingId: String, + version: Option[Long] = None, + useCache: Boolean = true, + applyUpdates: Boolean = false)(implicit tc: TokenContext, ec: ExecutionContext): Fox[VolumeTracing] = + if (tracingId == TracingIds.dummyTracingId) + Fox.successful(volumeTracingService.dummyTracing) + else { + for { + annotation <- getWithTracings(annotationId, version, List.empty, List(tracingId), requestAll = false) // TODO is applyUpdates still needed? + tracing <- annotation.getVolume(tracingId) + } yield tracing + } + + def findSkeleton( + annotationId: String, + tracingId: String, + version: Option[Long] = None, + useCache: Boolean = true, + applyUpdates: Boolean = false)(implicit tc: TokenContext, ec: ExecutionContext): Fox[SkeletonTracing] = + if (tracingId == TracingIds.dummyTracingId) + Fox.successful(skeletonTracingService.dummyTracing) + else { + for { + annotation <- getWithTracings(annotationId, version, List(tracingId), List.empty, requestAll = false) // TODO is applyUpdates still needed? + tracing <- annotation.getSkeleton(tracingId) + } yield tracing + } + + def findMultipleVolumes(selectors: List[Option[TracingSelector]], + useCache: Boolean = true, + applyUpdates: Boolean = false)(implicit tc: TokenContext, + ec: ExecutionContext): Fox[List[Option[VolumeTracing]]] = + Fox.combined { + selectors.map { + case Some(selector) => + for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(selector.tracingId) + tracing <- findVolume(annotationId, selector.tracingId, selector.version, useCache, applyUpdates) + .map(Some(_)) + } yield tracing + case None => Fox.successful(None) + } + } + + def findMultipleSkeletons(selectors: List[Option[TracingSelector]], + useCache: Boolean = true, + applyUpdates: Boolean = false)(implicit tc: TokenContext, + ec: ExecutionContext): Fox[List[Option[SkeletonTracing]]] = + Fox.combined { + selectors.map { + case Some(selector) => + for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(selector.tracingId) + tracing <- findSkeleton(annotationId, selector.tracingId, selector.version, useCache, applyUpdates) + .map(Some(_)) + } yield tracing + case None => Fox.successful(None) + } + } + } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala index 96430c6e33f..8e5cc234a5e 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala @@ -41,7 +41,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- volumeTracingService.find(annotationId, tracingId) + tracing <- annotationService.findVolume(annotationId, tracingId) tracingMappingName <- tracing.mappingName ?~> "annotation.noMappingSet" _ <- assertMappingIsNotLocked(tracing) _ <- bool2Fox(volumeTracingService.volumeBucketsAreEmpty(tracingId)) ?~> "annotation.volumeBucketsNotEmpty" @@ -82,7 +82,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer Action.async(validateJson[List[UpdateActionGroup]]) { implicit request => accessTokenService.validateAccess(UserAccessRequest.writeTracing(tracingId)) { for { - tracing <- tracingService.find(annotationId, tracingId) + tracing <- annotationService.findVolume(annotationId, tracingId) mappingName <- tracing.mappingName.toFox _ <- editableMappingService.assertTracingHasEditableMapping(tracing) currentVersion <- editableMappingService.getClosestMaterializableVersionOrZero(mappingName, None) @@ -111,7 +111,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- volumeTracingService.find(annotationId, tracingId) + tracing <- annotationService.findVolume(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) editableMappingInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId, version) infoJson = editableMappingService.infoJson(tracingId = tracingId, editableMappingInfo = editableMappingInfo) @@ -126,7 +126,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- volumeTracingService.find(annotationId, tracingId) + tracing <- annotationService.findVolume(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) agglomerateGraphBox: Box[AgglomerateGraph] <- editableMappingService .getAgglomerateGraphForId(tracingId, tracing.version, agglomerateId) @@ -148,7 +148,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- volumeTracingService.find(annotationId, tracingId) + tracing <- annotationService.findVolume(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) editableMappingInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId, version = None) @@ -170,7 +170,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- volumeTracingService.find(annotationId, tracingId) + tracing <- annotationService.findVolume(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) editableMappingInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId) @@ -190,7 +190,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- volumeTracingService.find(annotationId, tracingId) + tracing <- annotationService.findVolume(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) editableMappingInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId) @@ -209,7 +209,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- volumeTracingService.find(annotationId, tracingId) + tracing <- annotationService.findVolume(annotationId, tracingId) _ <- bool2Fox(tracing.getHasEditableMapping) ?~> "Cannot query agglomerate skeleton for volume annotation" editableMappingInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId) remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala index f74c54dc999..fe1e105568e 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala @@ -6,23 +6,28 @@ import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.SkeletonTracing.{SkeletonTracing, SkeletonTracingOpt, SkeletonTracings} import com.scalableminds.webknossos.datastore.services.UserAccessRequest import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService +import com.scalableminds.webknossos.tracingstore.tracings.TracingSelector import com.scalableminds.webknossos.tracingstore.tracings.skeleton._ import com.scalableminds.webknossos.tracingstore.tracings.volume.MergedVolumeStats import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingStoreAccessTokenService} -import net.liftweb.common.Empty +import net.liftweb.common.{Empty, Failure, Full} import play.api.i18n.Messages import play.api.libs.json.Json +import com.scalableminds.webknossos.datastore.controllers.Controller import play.api.mvc.{Action, AnyContent, PlayBodyParsers} +import com.scalableminds.util.tools.JsonHelper.{boxFormat, optionFormat} +import com.scalableminds.webknossos.tracingstore.annotation.TSAnnotationService import scala.concurrent.ExecutionContext -class SkeletonTracingController @Inject()(val tracingService: SkeletonTracingService, - val remoteWebknossosClient: TSRemoteWebknossosClient, - val accessTokenService: TracingStoreAccessTokenService, - val slackNotificationService: TSSlackNotificationService)( +class SkeletonTracingController @Inject()(skeletonTracingService: SkeletonTracingService, + remoteWebknossosClient: TSRemoteWebknossosClient, + annotationService: TSAnnotationService, + accessTokenService: TracingStoreAccessTokenService, + slackNotificationService: TSSlackNotificationService)( implicit val ec: ExecutionContext, val bodyParsers: PlayBodyParsers) - extends TracingController[SkeletonTracing, SkeletonTracings] { + extends Controller { implicit val tracingsCompanion: SkeletonTracings.type = SkeletonTracings @@ -35,15 +40,104 @@ class SkeletonTracingController @Inject()(val tracingService: SkeletonTracingSer implicit def unpackMultiple(tracings: SkeletonTracings): List[Option[SkeletonTracing]] = tracings.tracings.toList.map(_.tracing) + def save(): Action[SkeletonTracing] = Action.async(validateProto[SkeletonTracing]) { implicit request => + log() { + logTime(slackNotificationService.noticeSlowRequest) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { + val tracing = request.body + skeletonTracingService.save(tracing, None, 0).map { newId => + Ok(Json.toJson(newId)) + } + } + } + } + } + + def saveMultiple(): Action[SkeletonTracings] = Action.async(validateProto[SkeletonTracings]) { implicit request => + log() { + logTime(slackNotificationService.noticeSlowRequest) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { + val savedIds = Fox.sequence(request.body.map { tracingOpt: Option[SkeletonTracing] => + tracingOpt match { + case Some(tracing) => skeletonTracingService.save(tracing, None, 0).map(Some(_)) + case _ => Fox.successful(None) + } + }) + savedIds.map(id => Ok(Json.toJson(id))) + } + } + } + } + + def get(tracingId: String, version: Option[Long]): Action[AnyContent] = + Action.async { implicit request => + log() { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { + for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) + tracing <- annotationService.findSkeleton(annotationId, tracingId, version, applyUpdates = true) ?~> Messages( + "tracing.notFound") + } yield Ok(tracing.toByteArray).as(protobufMimeType) + } + } + } + + def getMultiple: Action[List[Option[TracingSelector]]] = + Action.async(validateJson[List[Option[TracingSelector]]]) { implicit request => + log() { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { + for { + tracings <- annotationService.findMultipleSkeletons(request.body, applyUpdates = true) + } yield { + Ok(tracings.toByteArray).as(protobufMimeType) + } + } + } + } + + def mergedFromIds(persist: Boolean): Action[List[Option[TracingSelector]]] = + Action.async(validateJson[List[Option[TracingSelector]]]) { implicit request => + log() { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { + for { + tracingOpts <- annotationService.findMultipleSkeletons(request.body, applyUpdates = true) ?~> Messages( + "tracing.notFound") + tracingsWithIds = tracingOpts.zip(request.body).flatMap { + case (Some(tracing), Some(selector)) => Some((tracing, selector.tracingId)) + case _ => None + } + newTracingId = skeletonTracingService.generateTracingId + mergedVolumeStats <- skeletonTracingService.mergeVolumeData(request.body.flatten, + tracingsWithIds.map(_._1), + newTracingId, + newVersion = 0L, + toCache = !persist) + mergeEditableMappingsResultBox <- skeletonTracingService + .mergeEditableMappings(newTracingId, tracingsWithIds) + .futureBox + newEditableMappingIdOpt <- mergeEditableMappingsResultBox match { + case Full(()) => Fox.successful(Some(newTracingId)) + case Empty => Fox.successful(None) + case f: Failure => f.toFox + } + mergedTracing <- Fox.box2Fox( + skeletonTracingService.merge(tracingsWithIds.map(_._1), mergedVolumeStats, newEditableMappingIdOpt)) + _ <- skeletonTracingService.save(mergedTracing, Some(newTracingId), version = 0, toCache = !persist) + } yield Ok(Json.toJson(newTracingId)) + } + } + } + def mergedFromContents(persist: Boolean): Action[SkeletonTracings] = Action.async(validateProto[SkeletonTracings]) { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { val tracings: List[Option[SkeletonTracing]] = request.body for { - mergedTracing <- Fox.box2Fox(tracingService.merge(tracings.flatten, MergedVolumeStats.empty(), Empty)) - processedTracing = tracingService.remapTooLargeTreeIds(mergedTracing) - newId <- tracingService.save(processedTracing, None, processedTracing.version, toCache = !persist) + mergedTracing <- Fox.box2Fox( + skeletonTracingService.merge(tracings.flatten, MergedVolumeStats.empty(), Empty)) + processedTracing = skeletonTracingService.remapTooLargeTreeIds(mergedTracing) + newId <- skeletonTracingService.save(processedTracing, None, processedTracing.version, toCache = !persist) } yield Ok(Json.toJson(newId)) } } @@ -60,19 +154,17 @@ class SkeletonTracingController @Inject()(val tracingService: SkeletonTracingSer accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId, version, applyUpdates = true) ?~> Messages( + tracing <- annotationService.findSkeleton(annotationId, tracingId, version, applyUpdates = true) ?~> Messages( "tracing.notFound") editPositionParsed <- Fox.runOptional(editPosition)(Vec3Int.fromUriLiteral) editRotationParsed <- Fox.runOptional(editRotation)(Vec3Double.fromUriLiteral) boundingBoxParsed <- Fox.runOptional(boundingBox)(BoundingBox.fromLiteral) - newId <- tracingService.duplicate(tracing, - fromTask.getOrElse(false), - editPositionParsed, - editRotationParsed, - boundingBoxParsed) - } yield { - Ok(Json.toJson(newId)) - } + newId <- skeletonTracingService.duplicate(tracing, + fromTask.getOrElse(false), + editPositionParsed, + editRotationParsed, + boundingBoxParsed) + } yield Ok(Json.toJson(newId)) } } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala deleted file mode 100644 index a7ecc6fb09c..00000000000 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TracingController.scala +++ /dev/null @@ -1,131 +0,0 @@ -package com.scalableminds.webknossos.tracingstore.controllers - -import com.scalableminds.util.tools.Fox -import com.scalableminds.util.tools.JsonHelper.{boxFormat, optionFormat} -import com.scalableminds.webknossos.datastore.controllers.Controller -import com.scalableminds.webknossos.datastore.services.UserAccessRequest -import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService -import com.scalableminds.webknossos.tracingstore.tracings.{TracingSelector, TracingService} -import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingStoreAccessTokenService} -import net.liftweb.common.{Empty, Failure, Full} -import play.api.i18n.Messages -import play.api.libs.json.Json -import play.api.mvc.{Action, AnyContent, PlayBodyParsers} -import scalapb.{GeneratedMessage, GeneratedMessageCompanion} - -import scala.concurrent.ExecutionContext - -trait TracingController[T <: GeneratedMessage, Ts <: GeneratedMessage] extends Controller { - - def tracingService: TracingService[T] - - def remoteWebknossosClient: TSRemoteWebknossosClient - - def accessTokenService: TracingStoreAccessTokenService - - def slackNotificationService: TSSlackNotificationService - - implicit val tracingCompanion: GeneratedMessageCompanion[T] = tracingService.tracingCompanion - - implicit val tracingsCompanion: GeneratedMessageCompanion[Ts] - - implicit def unpackMultiple(tracings: Ts): List[Option[T]] - - implicit def packMultiple(tracings: List[T]): Ts - - implicit def packMultipleOpt(tracings: List[Option[T]]): Ts - - implicit val ec: ExecutionContext - - implicit val bodyParsers: PlayBodyParsers - - override def allowRemoteOrigin: Boolean = true - - def save(): Action[T] = Action.async(validateProto[T]) { implicit request => - log() { - logTime(slackNotificationService.noticeSlowRequest) { - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { - val tracing = request.body - tracingService.save(tracing, None, 0).map { newId => - Ok(Json.toJson(newId)) - } - } - } - } - } - - def saveMultiple(): Action[Ts] = Action.async(validateProto[Ts]) { implicit request => - log() { - logTime(slackNotificationService.noticeSlowRequest) { - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { - val savedIds = Fox.sequence(request.body.map { tracingOpt: Option[T] => - tracingOpt match { - case Some(tracing) => tracingService.save(tracing, None, 0).map(Some(_)) - case _ => Fox.successful(None) - } - }) - savedIds.map(id => Ok(Json.toJson(id))) - } - } - } - } - - def get(tracingId: String, version: Option[Long]): Action[AnyContent] = - Action.async { implicit request => - log() { - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { - for { - annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId, version, applyUpdates = true) ?~> Messages( - "tracing.notFound") - } yield Ok(tracing.toByteArray).as(protobufMimeType) - } - } - } - - def getMultiple: Action[List[Option[TracingSelector]]] = - Action.async(validateJson[List[Option[TracingSelector]]]) { implicit request => - log() { - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { - for { - tracings <- tracingService.findMultiple(request.body, applyUpdates = true) - } yield { - Ok(tracings.toByteArray).as(protobufMimeType) - } - } - } - } - - def mergedFromIds(persist: Boolean): Action[List[Option[TracingSelector]]] = - Action.async(validateJson[List[Option[TracingSelector]]]) { implicit request => - log() { - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { - for { - tracingOpts <- tracingService.findMultiple(request.body, applyUpdates = true) ?~> Messages( - "tracing.notFound") - tracingsWithIds = tracingOpts.zip(request.body).flatMap { - case (Some(tracing), Some(selector)) => Some((tracing, selector.tracingId)) - case _ => None - } - newTracingId = tracingService.generateTracingId - mergedVolumeStats <- tracingService.mergeVolumeData(request.body.flatten, - tracingsWithIds.map(_._1), - newTracingId, - newVersion = 0L, - toCache = !persist) - mergeEditableMappingsResultBox <- tracingService - .mergeEditableMappings(newTracingId, tracingsWithIds) - .futureBox - newEditableMappingIdOpt <- mergeEditableMappingsResultBox match { - case Full(()) => Fox.successful(Some(newTracingId)) - case Empty => Fox.successful(None) - case f: Failure => f.toFox - } - mergedTracing <- Fox.box2Fox( - tracingService.merge(tracingsWithIds.map(_._1), mergedVolumeStats, newEditableMappingIdOpt)) - _ <- tracingService.save(mergedTracing, Some(newTracingId), version = 0, toCache = !persist) - } yield Ok(Json.toJson(newTracingId)) - } - } - } -} diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 243761dea06..f84c7ef7d53 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -6,7 +6,9 @@ import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.ExtendedTypes.ExtendedString import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings} +import com.scalableminds.webknossos.datastore.controllers.Controller import com.scalableminds.webknossos.datastore.geometry.ListOfVec3IntProto +import com.scalableminds.util.tools.JsonHelper.{boxFormat, optionFormat} import com.scalableminds.webknossos.datastore.helpers.{ GetSegmentIndexParameters, ProtoGeometryImplicits, @@ -33,14 +35,14 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ VolumeSegmentStatisticsService, VolumeTracingService } -import com.scalableminds.webknossos.tracingstore.tracings.KeyValueStoreImplicits +import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingSelector} import com.scalableminds.webknossos.tracingstore.{ TSRemoteDatastoreClient, TSRemoteWebknossosClient, TracingStoreAccessTokenService, TracingStoreConfig } -import net.liftweb.common.Empty +import net.liftweb.common.{Empty, Failure, Full} import play.api.i18n.Messages import play.api.libs.Files.TemporaryFile import play.api.libs.json.Json @@ -51,7 +53,7 @@ import java.nio.{ByteBuffer, ByteOrder} import scala.concurrent.ExecutionContext class VolumeTracingController @Inject()( - val tracingService: VolumeTracingService, + val volumeTracingService: VolumeTracingService, val config: TracingStoreConfig, val remoteDataStoreClient: TSRemoteDatastoreClient, val accessTokenService: TracingStoreAccessTokenService, @@ -63,7 +65,7 @@ class VolumeTracingController @Inject()( volumeSegmentIndexService: VolumeSegmentIndexService, fullMeshService: TSFullMeshService, val rpc: RPC)(implicit val ec: ExecutionContext, val bodyParsers: PlayBodyParsers) - extends TracingController[VolumeTracing, VolumeTracings] + extends Controller with ProtoGeometryImplicits with KeyValueStoreImplicits { @@ -78,6 +80,94 @@ class VolumeTracingController @Inject()( implicit def unpackMultiple(tracings: VolumeTracings): List[Option[VolumeTracing]] = tracings.tracings.toList.map(_.tracing) + def save(): Action[VolumeTracing] = Action.async(validateProto[VolumeTracing]) { implicit request => + log() { + logTime(slackNotificationService.noticeSlowRequest) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { + val tracing = request.body + volumeTracingService.save(tracing, None, 0).map { newId => + Ok(Json.toJson(newId)) + } + } + } + } + } + + def saveMultiple(): Action[VolumeTracings] = Action.async(validateProto[VolumeTracings]) { implicit request => + log() { + logTime(slackNotificationService.noticeSlowRequest) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { + val savedIds = Fox.sequence(request.body.map { tracingOpt: Option[VolumeTracing] => + tracingOpt match { + case Some(tracing) => volumeTracingService.save(tracing, None, 0).map(Some(_)) + case _ => Fox.successful(None) + } + }) + savedIds.map(id => Ok(Json.toJson(id))) + } + } + } + } + + def get(tracingId: String, version: Option[Long]): Action[AnyContent] = + Action.async { implicit request => + log() { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { + for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) + tracing <- annotationService.findVolume(annotationId, tracingId, version, applyUpdates = true) ?~> Messages( + "tracing.notFound") + } yield Ok(tracing.toByteArray).as(protobufMimeType) + } + } + } + + def getMultiple: Action[List[Option[TracingSelector]]] = + Action.async(validateJson[List[Option[TracingSelector]]]) { implicit request => + log() { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { + for { + tracings <- annotationService.findMultipleVolumes(request.body, applyUpdates = true) + } yield { + Ok(tracings.toByteArray).as(protobufMimeType) + } + } + } + } + + def mergedFromIds(persist: Boolean): Action[List[Option[TracingSelector]]] = + Action.async(validateJson[List[Option[TracingSelector]]]) { implicit request => + log() { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { + for { + tracingOpts <- annotationService.findMultipleVolumes(request.body, applyUpdates = true) ?~> Messages( + "tracing.notFound") + tracingsWithIds = tracingOpts.zip(request.body).flatMap { + case (Some(tracing), Some(selector)) => Some((tracing, selector.tracingId)) + case _ => None + } + newTracingId = volumeTracingService.generateTracingId + mergedVolumeStats <- volumeTracingService.mergeVolumeData(request.body.flatten, + tracingsWithIds.map(_._1), + newTracingId, + newVersion = 0L, + toCache = !persist) + mergeEditableMappingsResultBox <- volumeTracingService + .mergeEditableMappings(newTracingId, tracingsWithIds) + .futureBox + newEditableMappingIdOpt <- mergeEditableMappingsResultBox match { + case Full(()) => Fox.successful(Some(newTracingId)) + case Empty => Fox.successful(None) + case f: Failure => f.toFox + } + mergedTracing <- Fox.box2Fox( + volumeTracingService.merge(tracingsWithIds.map(_._1), mergedVolumeStats, newEditableMappingIdOpt)) + _ <- volumeTracingService.save(mergedTracing, Some(newTracingId), version = 0, toCache = !persist) + } yield Ok(Json.toJson(newTracingId)) + } + } + } + def initialData(tracingId: String, minResolution: Option[Int], maxResolution: Option[Int]): Action[AnyContent] = Action.async { implicit request => log() { @@ -86,12 +176,12 @@ class VolumeTracingController @Inject()( for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) initialData <- request.body.asRaw.map(_.asFile) ?~> Messages("zipFile.notFound") - tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") resolutionRestrictions = ResolutionRestrictions(minResolution, maxResolution) - resolutions <- tracingService + resolutions <- volumeTracingService .initializeWithData(annotationId, tracingId, tracing, initialData, resolutionRestrictions) .toFox - _ <- tracingService.updateResolutionList(tracingId, tracing, resolutions) + _ <- volumeTracingService.updateResolutionList(tracingId, tracing, resolutions) } yield Ok(Json.toJson(tracingId)) } } @@ -106,12 +196,14 @@ class VolumeTracingController @Inject()( _ <- Fox.successful(()) tracings = request.body shouldCreateSegmentIndex = volumeSegmentIndexService.shouldCreateSegmentIndexForMerged(tracings.flatten) - mt <- tracingService.merge(tracings.flatten, MergedVolumeStats.empty(shouldCreateSegmentIndex), Empty).toFox + mt <- volumeTracingService + .merge(tracings.flatten, MergedVolumeStats.empty(shouldCreateSegmentIndex), Empty) + .toFox // segment lists for multi-volume uploads are not supported yet, compare https://github.com/scalableminds/webknossos/issues/6887 mergedTracing = mt.copy(segments = List.empty) - newId <- tracingService.save(mergedTracing, None, mergedTracing.version, toCache = !persist) + newId <- volumeTracingService.save(mergedTracing, None, mergedTracing.version, toCache = !persist) } yield Ok(Json.toJson(newId)) } } @@ -125,11 +217,11 @@ class VolumeTracingController @Inject()( for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) initialData <- request.body.asRaw.map(_.asFile) ?~> Messages("zipFile.notFound") - tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") - resolutions <- tracingService + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") + resolutions <- volumeTracingService .initializeWithDataMultiple(annotationId, tracingId, tracing, initialData) .toFox - _ <- tracingService.updateResolutionList(tracingId, tracing, resolutions) + _ <- volumeTracingService.updateResolutionList(tracingId, tracing, resolutions) } yield Ok(Json.toJson(tracingId)) } } @@ -146,13 +238,13 @@ class VolumeTracingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId, version) ?~> Messages("tracing.notFound") + tracing <- annotationService.findVolume(annotationId, tracingId, version) ?~> Messages("tracing.notFound") volumeDataZipFormatParsed <- VolumeDataZipFormat.fromString(volumeDataZipFormat).toFox voxelSizeFactorParsedOpt <- Fox.runOptional(voxelSizeFactor)(Vec3Double.fromUriLiteral) voxelSizeUnitParsedOpt <- Fox.runOptional(voxelSizeUnit)(LengthUnit.fromString) voxelSize = voxelSizeFactorParsedOpt.map(voxelSizeParsed => VoxelSize.fromFactorAndUnitWithDefault(voxelSizeParsed, voxelSizeUnitParsedOpt)) - data <- tracingService.allDataZip( + data <- volumeTracingService.allDataZip( tracingId, tracing, volumeDataZipFormatParsed, @@ -169,11 +261,11 @@ class VolumeTracingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") (data, indices) <- if (tracing.getHasEditableMapping) { val mappingLayer = annotationService.editableMappingLayer(annotationId, tracingId, tracing) editableMappingService.volumeData(mappingLayer, request.body) - } else tracingService.data(tracingId, tracing, request.body) + } else volumeTracingService.data(tracingId, tracing, request.body) } yield Ok(data).withHeaders(getMissingBucketsHeaders(indices): _*) } } @@ -198,7 +290,7 @@ class VolumeTracingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") _ = logger.info(s"Duplicating volume tracing $tracingId...") datasetBoundingBox = request.body.asJson.flatMap(_.validateOpt[BoundingBox].asOpt.flatten) resolutionRestrictions = ResolutionRestrictions(minResolution, maxResolution) @@ -206,12 +298,12 @@ class VolumeTracingController @Inject()( editRotationParsed <- Fox.runOptional(editRotation)(Vec3Double.fromUriLiteral) boundingBoxParsed <- Fox.runOptional(boundingBox)(BoundingBox.fromLiteral) remoteFallbackLayerOpt <- Fox.runIf(tracing.getHasEditableMapping)( - tracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId)) - newTracingId = tracingService.generateTracingId + volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId)) + newTracingId = volumeTracingService.generateTracingId // TODO /*_ <- Fox.runIf(tracing.getHasEditableMapping)( editableMappingService.duplicate(tracingId, newTracingId, version = None, remoteFallbackLayerOpt))*/ - (newId, newTracing) <- tracingService.duplicate( + (newId, newTracing) <- volumeTracingService.duplicate( annotationId, tracingId, newTracingId, @@ -224,7 +316,8 @@ class VolumeTracingController @Inject()( boundingBoxParsed, mappingName = None ) - _ <- Fox.runIfOptionTrue(downsample)(tracingService.downsample(annotationId, newId, tracingId, newTracing)) + _ <- Fox.runIfOptionTrue(downsample)( + volumeTracingService.downsample(annotationId, newId, tracingId, newTracing)) } yield Ok(Json.toJson(newId)) } } @@ -237,14 +330,14 @@ class VolumeTracingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") currentVersion <- request.body.dataParts("currentVersion").headOption.flatMap(_.toIntOpt).toFox zipFile <- request.body.files.headOption.map(f => new File(f.ref.path.toString)).toFox - largestSegmentId <- tracingService.importVolumeData(annotationId, - tracingId, - tracing, - zipFile, - currentVersion) + largestSegmentId <- volumeTracingService.importVolumeData(annotationId, + tracingId, + tracing, + zipFile, + currentVersion) } yield Ok(Json.toJson(largestSegmentId)) } } @@ -256,11 +349,11 @@ class VolumeTracingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") currentVersion <- annotationService.currentMaterializableVersion(tracingId) before = Instant.now - canAddSegmentIndex <- tracingService.checkIfSegmentIndexMayBeAdded(tracingId, tracing) - processedBucketCountOpt <- Fox.runIf(canAddSegmentIndex)(tracingService + canAddSegmentIndex <- volumeTracingService.checkIfSegmentIndexMayBeAdded(tracingId, tracing) + processedBucketCountOpt <- Fox.runIf(canAddSegmentIndex)(volumeTracingService .addSegmentIndex(annotationId, tracingId, tracing, currentVersion, dryRun)) ?~> "addSegmentIndex.failed" currentVersionNew <- annotationService.currentMaterializableVersion(tracingId) _ <- Fox.runIf(!dryRun)(bool2Fox( @@ -283,11 +376,11 @@ class VolumeTracingController @Inject()( // consecutive 3D points (i.e., nine floats) form a triangle. // There are no shared vertices between triangles. annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") (vertices: Array[Float], neighbors: List[Int]) <- if (tracing.getHasEditableMapping) { val editableMappingLayer = annotationService.editableMappingLayer(annotationId, tracingId, tracing) editableMappingService.createAdHocMesh(editableMappingLayer, request.body) - } else tracingService.createAdHocMesh(tracingId, tracing, request.body) + } else volumeTracingService.createAdHocMesh(tracingId, tracing, request.body) } yield { // We need four bytes for each float val responseBuffer = ByteBuffer.allocate(vertices.length * 4).order(ByteOrder.LITTLE_ENDIAN) @@ -317,7 +410,8 @@ class VolumeTracingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - positionOpt <- tracingService.findData(annotationId, tracingId) + tracing <- annotationService.findVolume(annotationId, tracingId) + positionOpt <- volumeTracingService.findData(tracingId, tracing) } yield { Ok(Json.obj("position" -> positionOpt, "resolution" -> positionOpt.map(_ => Vec3Int.ones))) } @@ -329,7 +423,7 @@ class VolumeTracingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId) + tracing <- annotationService.findVolume(annotationId, tracingId) mappingName <- annotationService.baseMappingName(annotationId, tracingId, tracing) segmentVolumes <- Fox.serialCombined(request.body.segmentIds) { segmentId => volumeSegmentStatisticsService.getSegmentVolume(annotationId, @@ -348,7 +442,7 @@ class VolumeTracingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId) + tracing <- annotationService.findVolume(annotationId, tracingId) mappingName <- annotationService.baseMappingName(annotationId, tracingId, tracing) segmentBoundingBoxes: List[BoundingBox] <- Fox.serialCombined(request.body.segmentIds) { segmentId => volumeSegmentStatisticsService.getSegmentBoundingBox(annotationId, @@ -367,8 +461,8 @@ class VolumeTracingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - fallbackLayer <- tracingService.getFallbackLayer(annotationId, tracingId) - tracing <- tracingService.find(annotationId, tracingId) + tracing <- annotationService.findVolume(annotationId, tracingId) + fallbackLayer <- volumeTracingService.getFallbackLayer(tracingId, tracing) mappingName <- annotationService.baseMappingName(annotationId, tracingId, tracing) _ <- bool2Fox(DataLayer.bucketSize <= request.body.cubeSize) ?~> "cubeSize must be at least one bucket (32³)" bucketPositionsRaw: ListOfVec3IntProto <- volumeSegmentIndexService @@ -380,7 +474,7 @@ class VolumeTracingController @Inject()( additionalCoordinates = request.body.additionalCoordinates, additionalAxes = AdditionalAxis.fromProtosAsOpt(tracing.additionalAxes), mappingName = mappingName, - editableMappingTracingId = tracingService.editableMappingTracingId(tracing, tracingId) + editableMappingTracingId = volumeTracingService.editableMappingTracingId(tracing, tracingId) ) bucketPositionsForCubeSize = bucketPositionsRaw.values .map(vec3IntFromProto) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala index 7108a8104f6..2e7012c355b 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingZarrStreamingController.scala @@ -64,7 +64,7 @@ class VolumeTracingZarrStreamingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) additionalFiles = if (zarrVersion == 2) List(NgffMetadata.FILENAME_DOT_ZATTRS, NgffGroupHeader.FILENAME_DOT_ZGROUP) @@ -84,7 +84,7 @@ class VolumeTracingZarrStreamingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto(_).toMagLiteral(allowScalar = true)) additionalFiles = if (zarrVersion == 2) List(NgffMetadata.FILENAME_DOT_ZATTRS, NgffGroupHeader.FILENAME_DOT_ZGROUP) @@ -98,7 +98,7 @@ class VolumeTracingZarrStreamingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND @@ -118,7 +118,7 @@ class VolumeTracingZarrStreamingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND @@ -132,7 +132,7 @@ class VolumeTracingZarrStreamingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND _ <- bool2Fox(existingMags.contains(magParsed)) ?~> Messages("tracing.wrongMag", tracingId, mag) ~> NOT_FOUND @@ -167,7 +167,7 @@ class VolumeTracingZarrStreamingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND @@ -226,7 +226,7 @@ class VolumeTracingZarrStreamingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) dataSource <- remoteWebknossosClient.getDataSourceForTracing(tracingId) ~> NOT_FOUND omeNgffHeader = NgffMetadata.fromNameVoxelSizeAndMags(tracingId, @@ -242,7 +242,7 @@ class VolumeTracingZarrStreamingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) dataSource <- remoteWebknossosClient.getDataSourceForTracing(tracingId) ~> NOT_FOUND omeNgffHeader = NgffMetadataV0_5.fromNameVoxelSizeAndMags(tracingId, @@ -259,7 +259,7 @@ class VolumeTracingZarrStreamingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND zarrLayer = ZarrSegmentationLayer( name = tracingName.getOrElse(tracingId), largestSegmentId = tracing.largestSegmentId, @@ -280,7 +280,7 @@ class VolumeTracingZarrStreamingController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { for { annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- tracingService.find(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") ~> NOT_FOUND existingMags = tracing.resolutions.map(vec3IntFromProto) magParsed <- Vec3Int.fromMagLiteral(mag, allowScalar = true) ?~> Messages("dataLayer.invalidMag", mag) ~> NOT_FOUND diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala index 58cb7da7d16..afe6fd032a4 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala @@ -3,7 +3,6 @@ package com.scalableminds.webknossos.tracingstore.tracings import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingStoreRedisStore} -import com.scalableminds.webknossos.tracingstore.annotation.TSAnnotationService import com.scalableminds.webknossos.tracingstore.tracings.TracingType.TracingType import com.scalableminds.webknossos.tracingstore.tracings.volume.MergedVolumeStats import com.typesafe.scalalogging.LazyLogging @@ -40,8 +39,6 @@ trait TracingService[T <: GeneratedMessage] def tracingMigrationService: TracingMigrationService[T] - def annotationService: TSAnnotationService - def dummyTracing: T implicit def tracingCompanion: GeneratedMessageCompanion[T] @@ -68,25 +65,6 @@ trait TracingService[T <: GeneratedMessage] } */ - def find(annotationId: String, - tracingId: String, - version: Option[Long] = None, - useCache: Boolean = true, - applyUpdates: Boolean = false)(implicit tc: TokenContext): Fox[T] - - def findMultiple(selectors: List[Option[TracingSelector]], useCache: Boolean = true, applyUpdates: Boolean = false)( - implicit tc: TokenContext): Fox[List[Option[T]]] = - Fox.combined { - selectors.map { - case Some(selector) => - for { - annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(selector.tracingId) - tracing <- find(annotationId, selector.tracingId, selector.version, useCache, applyUpdates).map(Some(_)) - } yield tracing - case None => Fox.successful(None) - } - } - def generateTracingId: String = UUID.randomUUID.toString def save(tracing: T, tracingId: Option[String], version: Long, toCache: Boolean = false): Fox[String] = { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala index 0512226823f..dcbe1eb86c0 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala @@ -9,7 +9,6 @@ import com.scalableminds.webknossos.datastore.geometry.NamedBoundingBoxProto import com.scalableminds.webknossos.datastore.helpers.{ProtoGeometryImplicits, SkeletonTracingDefaults} import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingStoreRedisStore} -import com.scalableminds.webknossos.tracingstore.annotation.TSAnnotationService import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.webknossos.tracingstore.tracings.volume.MergedVolumeStats import net.liftweb.common.{Box, Full} @@ -24,7 +23,6 @@ class SkeletonTracingService @Inject()( val temporaryTracingIdStore: TracingStoreRedisStore, val remoteWebknossosClient: TSRemoteWebknossosClient, val uncommittedUpdatesStore: TracingStoreRedisStore, - val annotationService: TSAnnotationService, val tracingMigrationService: SkeletonTracingMigrationService)(implicit val ec: ExecutionContext) extends TracingService[SkeletonTracing] with KeyValueStoreImplicits @@ -37,24 +35,6 @@ class SkeletonTracingService @Inject()( implicit val tracingCompanion: SkeletonTracing.type = SkeletonTracing - def find(annotationId: String, - tracingId: String, - version: Option[Long] = None, - useCache: Boolean = true, - applyUpdates: Boolean = false)(implicit tc: TokenContext): Fox[SkeletonTracing] = - if (tracingId == TracingIds.dummyTracingId) - Fox.successful(dummyTracing) - else { - for { - annotation <- annotationService.getWithTracings(annotationId, - version, - List(tracingId), - List.empty, - requestAll = false) // TODO is applyUpdates still needed? - tracing <- annotation.getSkeleton(tracingId) - } yield tracing - } - def duplicate(tracing: SkeletonTracing, fromTask: Boolean, editPosition: Option[Vec3Int], diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala index 541b0ed519a..630b587c9b4 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/TSFullMeshService.scala @@ -40,7 +40,7 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, implicit ec: ExecutionContext, tc: TokenContext): Fox[Array[Byte]] = for { - tracing <- volumeTracingService.find(annotationId, tracingId) ?~> "tracing.notFound" + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> "tracing.notFound" data <- if (fullMeshRequest.meshFileName.isDefined) loadFullMeshFromMeshfile(annotationId, tracingId, tracing, fullMeshRequest) else loadFullMeshFromAdHoc(annotationId, tracingId, tracing, fullMeshRequest) @@ -98,7 +98,7 @@ class TSFullMeshService @Inject()(volumeTracingService: VolumeTracingService, voxelSize: VoxelSize, fullMeshRequest: FullMeshRequest)(implicit ec: ExecutionContext, tc: TokenContext): Fox[List[Array[Float]]] = for { - fallbackLayer <- volumeTracingService.getFallbackLayer(annotationId, tracingId) + fallbackLayer <- volumeTracingService.getFallbackLayer(tracingId, tracing) mappingName <- annotationService.baseMappingName(annotationId, tracingId, tracing) bucketPositionsRaw: ListOfVec3IntProto <- volumeSegmentIndexService .getSegmentToBucketIndexWithEmptyFallbackWithoutBuffer( diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentStatisticsService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentStatisticsService.scala index 82109a461a5..cede715f841 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentStatisticsService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentStatisticsService.scala @@ -58,9 +58,9 @@ class VolumeSegmentStatisticsService @Inject()(volumeTracingService: VolumeTraci private def getTypedDataForBucketPosition(annotationId: String, tracingId: String)( bucketPosition: Vec3Int, mag: Vec3Int, - additionalCoordinates: Option[Seq[AdditionalCoordinate]])(implicit tc: TokenContext) = + additionalCoordinates: Option[Seq[AdditionalCoordinate]])(implicit tc: TokenContext, ec: ExecutionContext) = for { - tracing <- volumeTracingService.find(annotationId, tracingId) ?~> "tracing.notFound" + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> "tracing.notFound" bucketData <- getVolumeDataForPositions(annotationId, tracingId, tracing, @@ -79,8 +79,8 @@ class VolumeSegmentStatisticsService @Inject()(volumeTracingService: VolumeTraci segmentId: Long, mag: Vec3Int)(implicit ec: ExecutionContext, tc: TokenContext) = for { - fallbackLayer <- volumeTracingService.getFallbackLayer(annotationId, tracingId) - tracing <- volumeTracingService.find(annotationId, tracingId) ?~> "tracing.notFound" + tracing <- annotationService.findVolume(annotationId, tracingId) ?~> "tracing.notFound" + fallbackLayer <- volumeTracingService.getFallbackLayer(tracingId, tracing) additionalAxes = AdditionalAxis.fromProtosAsOpt(tracing.additionalAxes) allBucketPositions: ListOfVec3IntProto <- volumeSegmentIndexService .getSegmentToBucketIndexWithEmptyFallbackWithoutBuffer( diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala index 5a85c53a8f8..1b25a1c7815 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala @@ -82,16 +82,16 @@ trait VolumeTracingDownsampling annotationId: String, tracingId: String, oldTracingId: String, - tracing: VolumeTracing, + newTracing: VolumeTracing, dataLayer: VolumeTracingLayer, tracingService: VolumeTracingService)(implicit ec: ExecutionContext, tc: TokenContext): Fox[List[Vec3Int]] = { val bucketVolume = 32 * 32 * 32 for { - _ <- bool2Fox(tracing.version == 0L) ?~> "Tracing has already been edited." - _ <- bool2Fox(tracing.resolutions.nonEmpty) ?~> "Cannot downsample tracing with no resolution list" - sourceMag = getSourceMag(tracing) - magsToCreate <- getMagsToCreate(tracing, oldTracingId) - elementClass = elementClassFromProto(tracing.elementClass) + _ <- bool2Fox(newTracing.version == 0L) ?~> "Tracing has already been edited." + _ <- bool2Fox(newTracing.resolutions.nonEmpty) ?~> "Cannot downsample tracing with no resolution list" + sourceMag = getSourceMag(newTracing) + magsToCreate <- getMagsToCreate(newTracing, oldTracingId) + elementClass = elementClassFromProto(newTracing.elementClass) bucketDataMapMutable = new mutable.HashMap[BucketPosition, Array[Byte]]().withDefault(_ => revertedValue) _ = fillMapWithSourceBucketsInplace(bucketDataMapMutable, tracingId, dataLayer, sourceMag) originalBucketPositions = bucketDataMapMutable.keys.toList @@ -107,28 +107,27 @@ trait VolumeTracingDownsampling dataLayer) requiredMag } - fallbackLayer <- tracingService.getFallbackLayer(annotationId, oldTracingId) // remote wk does not know the new id yet - tracing <- tracingService.find(annotationId, tracingId) ?~> "tracing.notFound" + fallbackLayer <- tracingService.getFallbackLayer(oldTracingId, newTracing) // remote wk does not know the new id yet segmentIndexBuffer = new VolumeSegmentIndexBuffer(tracingId, volumeSegmentIndexClient, - tracing.version, + newTracing.version, tracingService.remoteDatastoreClient, fallbackLayer, dataLayer.additionalAxes, tc) _ <- Fox.serialCombined(updatedBucketsMutable.toList) { bucketPosition: BucketPosition => for { - _ <- saveBucket(dataLayer, bucketPosition, bucketDataMapMutable(bucketPosition), tracing.version) - mappingName <- selectMappingName(tracing) - _ <- Fox.runIfOptionTrue(tracing.hasSegmentIndex)( + _ <- saveBucket(dataLayer, bucketPosition, bucketDataMapMutable(bucketPosition), newTracing.version) + mappingName <- selectMappingName(newTracing) + _ <- Fox.runIfOptionTrue(newTracing.hasSegmentIndex)( updateSegmentIndex( segmentIndexBuffer, bucketPosition, bucketDataMapMutable(bucketPosition), Empty, - tracing.elementClass, + newTracing.elementClass, mappingName, - editableMappingTracingId(tracing, tracingId) + editableMappingTracingId(newTracing, tracingId) )) } yield () } @@ -259,14 +258,16 @@ trait VolumeTracingDownsampling private def getSourceMag(tracing: VolumeTracing): Vec3Int = tracing.resolutions.minBy(_.maxDim) - private def getMagsToCreate(tracing: VolumeTracing, oldTracingId: String): Fox[List[Vec3Int]] = + private def getMagsToCreate(tracing: VolumeTracing, oldTracingId: String)( + implicit tc: TokenContext): Fox[List[Vec3Int]] = for { requiredMags <- getRequiredMags(tracing, oldTracingId) sourceMag = getSourceMag(tracing) magsToCreate = requiredMags.filter(_.maxDim > sourceMag.maxDim) } yield magsToCreate - private def getRequiredMags(tracing: VolumeTracing, oldTracingId: String): Fox[List[Vec3Int]] = + private def getRequiredMags(tracing: VolumeTracing, oldTracingId: String)( + implicit tc: TokenContext): Fox[List[Vec3Int]] = for { dataSource: DataSourceLike <- tracingStoreWkRpcClient.getDataSourceForTracing(oldTracingId) magsForTracing = VolumeTracingDownsampling.magsForVolumeTracingByLayerName(dataSource, tracing.fallbackLayer) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index bf8b16b1bb3..1315db091ff 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -23,7 +23,7 @@ import com.scalableminds.webknossos.datastore.models.{ WebknossosAdHocMeshRequest } import com.scalableminds.webknossos.datastore.services._ -import com.scalableminds.webknossos.tracingstore.annotation.{TSAnnotationService, UpdateActionGroup} +import com.scalableminds.webknossos.tracingstore.annotation.UpdateActionGroup import com.scalableminds.webknossos.tracingstore.tracings.TracingType.TracingType import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeDataZipFormat.VolumeDataZipFormat @@ -56,7 +56,6 @@ class VolumeTracingService @Inject()( val uncommittedUpdatesStore: TracingStoreRedisStore, val temporaryTracingIdStore: TracingStoreRedisStore, val remoteDatastoreClient: TSRemoteDatastoreClient, - val annotationService: TSAnnotationService, val remoteWebknossosClient: TSRemoteWebknossosClient, val temporaryFileCreator: TemporaryFileCreator, val tracingMigrationService: VolumeTracingMigrationService, @@ -89,8 +88,8 @@ class VolumeTracingService @Inject()( adHocMeshServiceHolder.tracingStoreAdHocMeshConfig = (binaryDataService, 30 seconds, 1) val adHocMeshService: AdHocMeshService = adHocMeshServiceHolder.tracingStoreAdHocMeshService - private val fallbackLayerCache: AlfuCache[(String, String, Option[String]), Option[RemoteFallbackLayer]] = AlfuCache( - maxCapacity = 100) + private val fallbackLayerCache: AlfuCache[(String, Option[String], Option[String]), Option[RemoteFallbackLayer]] = + AlfuCache(maxCapacity = 100) override protected def updateSegmentIndex( segmentIndexBuffer: VolumeSegmentIndexBuffer, @@ -108,15 +107,14 @@ class VolumeTracingService @Inject()( mappingName, editableMappingTracingId) ?~> "volumeSegmentIndex.update.failed" - def applyBucketMutatingActions(annotationId: String, + def applyBucketMutatingActions(tracingId: String, + tracing: VolumeTracing, updateActions: List[BucketMutatingVolumeUpdateAction], newVersion: Long)(implicit tc: TokenContext): Fox[Unit] = for { // warning, may be called multiple times with the same version number (due to transaction management). // frontend ensures that each bucket is only updated once per transaction - tracingId <- updateActions.headOption.map(_.actionTracingId).toFox - fallbackLayerOpt <- getFallbackLayer(annotationId, tracingId) - tracing <- find(annotationId, tracingId) ?~> "tracing.notFound" + fallbackLayerOpt <- getFallbackLayer(tracingId, tracing) segmentIndexBuffer = new VolumeSegmentIndexBuffer( tracingId, volumeSegmentIndexClient, @@ -137,7 +135,7 @@ class VolumeTracingService @Inject()( if (!tracing.getHasSegmentIndex) { Fox.failure("Cannot delete segment data for annotations without segment index.") } else - deleteSegmentData(annotationId, tracingId, tracing, a, segmentIndexBuffer, newVersion) ?~> "Failed to delete segment data." + deleteSegmentData(tracingId, tracing, a, segmentIndexBuffer, newVersion) ?~> "Failed to delete segment data." case _ => Fox.failure("Unknown bucket-mutating action.") } _ <- segmentIndexBuffer.flush() @@ -176,24 +174,6 @@ class VolumeTracingService @Inject()( } } yield volumeTracing - def find(annotationId: String, - tracingId: String, - version: Option[Long] = None, - useCache: Boolean = true, - applyUpdates: Boolean = false)(implicit tc: TokenContext): Fox[VolumeTracing] = - if (tracingId == TracingIds.dummyTracingId) - Fox.successful(dummyTracing) - else { - for { - annotation <- annotationService.getWithTracings(annotationId, - version, - List.empty, - List(tracingId), - requestAll = false) // TODO is applyUpdates still needed? - tracing <- annotation.getVolume(tracingId) - } yield tracing - } - override def editableMappingTracingId(tracing: VolumeTracing, tracingId: String): Option[String] = if (tracing.getHasEditableMapping) Some(tracingId) else None @@ -202,8 +182,7 @@ class VolumeTracingService @Inject()( Fox.failure("mappingName called on volumeTracing with editableMapping!") else Fox.successful(tracing.mappingName) - private def deleteSegmentData(annotationId: String, - tracingId: String, + private def deleteSegmentData(tracingId: String, volumeTracing: VolumeTracing, a: DeleteSegmentDataVolumeAction, segmentIndexBuffer: VolumeSegmentIndexBuffer, @@ -211,6 +190,7 @@ class VolumeTracingService @Inject()( for { _ <- Fox.successful(()) dataLayer = volumeTracingLayer(tracingId, volumeTracing) + fallbackLayer <- getFallbackLayer(tracingId, volumeTracing) possibleAdditionalCoordinates = AdditionalAxis.coordinateSpace(dataLayer.additionalAxes).map(Some(_)) additionalCoordinateList = if (possibleAdditionalCoordinates.isEmpty) { List(None) @@ -222,7 +202,6 @@ class VolumeTracingService @Inject()( Fox.serialCombined(additionalCoordinateList)(additionalCoordinates => { val mag = vec3IntFromProto(resolution) for { - fallbackLayer <- getFallbackLayer(annotationId, tracingId) bucketPositionsRaw <- volumeSegmentIndexService.getSegmentToBucketIndexWithEmptyFallbackWithoutBuffer( fallbackLayer, tracingId, @@ -374,7 +353,7 @@ class VolumeTracingService @Inject()( mergedVolume.largestSegmentId.toLong, tracing.elementClass) destinationDataLayer = volumeTracingLayer(tracingId, tracing) - fallbackLayer <- getFallbackLayer(annotationId, tracingId) + fallbackLayer <- getFallbackLayer(tracingId, tracing) segmentIndexBuffer = new VolumeSegmentIndexBuffer( tracingId, volumeSegmentIndexClient, @@ -415,7 +394,7 @@ class VolumeTracingService @Inject()( val dataLayer = volumeTracingLayer(tracingId, tracing) val savedResolutions = new mutable.HashSet[Vec3Int]() for { - fallbackLayer <- getFallbackLayer(annotationId, tracingId) + fallbackLayer <- getFallbackLayer(tracingId, tracing) mappingName <- selectMappingName(tracing) segmentIndexBuffer = new VolumeSegmentIndexBuffer( tracingId, @@ -522,7 +501,7 @@ class VolumeTracingService @Inject()( val tracingWithBB = addBoundingBoxFromTaskIfRequired(sourceTracing, fromTask, datasetBoundingBox) val tracingWithResolutionRestrictions = restrictMagList(tracingWithBB, resolutionRestrictions) for { - fallbackLayer <- getFallbackLayer(annotationId, tracingId) + fallbackLayer <- getFallbackLayer(tracingId, sourceTracing) hasSegmentIndex <- VolumeSegmentIndexService.canHaveSegmentIndex(remoteDatastoreClient, fallbackLayer) newTracing = tracingWithResolutionRestrictions.copy( createdTimestamp = System.currentTimeMillis(), @@ -568,7 +547,7 @@ class VolumeTracingService @Inject()( sourceDataLayer = volumeTracingLayer(sourceId, sourceTracing, isTemporaryTracing) buckets: Iterator[(BucketPosition, Array[Byte])] = sourceDataLayer.bucketProvider.bucketStream() destinationDataLayer = volumeTracingLayer(destinationId, destinationTracing) - fallbackLayer <- getFallbackLayer(annotationId, sourceId) + fallbackLayer <- getFallbackLayer(sourceId, sourceTracing) segmentIndexBuffer = new VolumeSegmentIndexBuffer( destinationId, volumeSegmentIndexClient, @@ -628,16 +607,16 @@ class VolumeTracingService @Inject()( toCache) } yield id - def downsample(annotationId: String, tracingId: String, oldTracingId: String, tracing: VolumeTracing)( + def downsample(annotationId: String, tracingId: String, oldTracingId: String, newTracing: VolumeTracing)( implicit tc: TokenContext): Fox[Unit] = for { resultingResolutions <- downsampleWithLayer(annotationId, tracingId, oldTracingId, - tracing, - volumeTracingLayer(tracingId, tracing), + newTracing, + volumeTracingLayer(tracingId, newTracing), this) - _ <- updateResolutionList(tracingId, tracing, resultingResolutions.toSet) + _ <- updateResolutionList(tracingId, newTracing, resultingResolutions.toSet) } yield () def volumeBucketsAreEmpty(tracingId: String): Boolean = @@ -660,9 +639,9 @@ class VolumeTracingService @Inject()( adHocMeshService.requestAdHocMeshViaActor(adHocMeshRequest) } - def findData(annotationId: String, tracingId: String)(implicit tc: TokenContext): Fox[Option[Vec3Int]] = + def findData(tracingId: String, tracing: VolumeTracing)(implicit tc: TokenContext): Fox[Option[Vec3Int]] = for { - tracing <- find(annotationId: String, tracingId) ?~> "tracing.notFound" + _ <- Fox.successful(()) volumeLayer = volumeTracingLayer(tracingId, tracing) bucketStream = volumeLayer.bucketProvider.bucketStream(Some(tracing.version)) bucketPosOpt = if (bucketStream.hasNext) { @@ -807,7 +786,7 @@ class VolumeTracingService @Inject()( elementClass) mergedAdditionalAxes <- Fox.box2Fox(AdditionalAxis.mergeAndAssertSameAdditionalAxes(tracings.map(t => AdditionalAxis.fromProtosAsOpt(t.additionalAxes)))) - fallbackLayer <- getFallbackLayer("dummyAnnotationId", tracingSelectors.head.tracingId) // TODO annotation id from selectors + fallbackLayer <- getFallbackLayer(tracingSelectors.head.tracingId, tracings.head) // TODO can we get rid of the head? segmentIndexBuffer = new VolumeSegmentIndexBuffer(newId, volumeSegmentIndexClient, newVersion, @@ -843,7 +822,7 @@ class VolumeTracingService @Inject()( isTemporaryTracing <- isTemporaryTracing(tracingId) sourceDataLayer = volumeTracingLayer(tracingId, tracing, isTemporaryTracing) buckets: Iterator[(BucketPosition, Array[Byte])] = sourceDataLayer.bucketProvider.bucketStream() - fallbackLayer <- getFallbackLayer(annotationId, tracingId) + fallbackLayer <- getFallbackLayer(tracingId, tracing) mappingName <- selectMappingName(tracing) segmentIndexBuffer = new VolumeSegmentIndexBuffer(tracingId, volumeSegmentIndexClient, @@ -919,7 +898,7 @@ class VolumeTracingService @Inject()( mergedVolume.largestSegmentId.toLong, tracing.elementClass) dataLayer = volumeTracingLayer(tracingId, tracing) - fallbackLayer <- getFallbackLayer(annotationId, tracingId) + fallbackLayer <- getFallbackLayer(tracingId, tracing) mappingName <- selectMappingName(tracing) segmentIndexBuffer <- Fox.successful( new VolumeSegmentIndexBuffer(tracingId, @@ -983,18 +962,17 @@ class VolumeTracingService @Inject()( Fox.failure("Cannot merge tracings with and without editable mappings") } - def getFallbackLayer(annotationId: String, tracingId: String)( + def getFallbackLayer(tracingId: String, tracing: VolumeTracing)( implicit tc: TokenContext): Fox[Option[RemoteFallbackLayer]] = - fallbackLayerCache.getOrLoad((annotationId, tracingId, tc.userTokenOpt), + fallbackLayerCache.getOrLoad((tracingId, tracing.fallbackLayer, tc.userTokenOpt), t => getFallbackLayerFromWebknossos(t._1, t._2)) - private def getFallbackLayerFromWebknossos(annotationId: String, tracingId: String)(implicit tc: TokenContext) = + private def getFallbackLayerFromWebknossos(tracingId: String, fallbackLayerName: Option[String])( + implicit tc: TokenContext) = Fox[Option[RemoteFallbackLayer]] { for { - tracing <- find(annotationId, tracingId) dataSource <- remoteWebknossosClient.getDataSourceForTracing(tracingId) dataSourceId = dataSource.id - fallbackLayerName = tracing.fallbackLayer fallbackLayer = dataSource.dataLayers .find(_.name == fallbackLayerName.getOrElse("")) .map(RemoteFallbackLayer.fromDataLayerAndDataSource(_, dataSourceId)) From 163354c761107199a75c5acb6ed2f02e7056914e Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 8 Oct 2024 15:07:29 +0200 Subject: [PATCH 094/150] revert volume buckets. TODO: lz4 exception? --- .../annotation/AnnotationReversion.scala | 17 ++++++++++++---- .../annotation/TSAnnotationService.scala | 16 +++++++-------- .../volume/VolumeTracingService.scala | 20 ++++++++----------- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala index c26f5bf7bfc..bb5bb384602 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala @@ -2,13 +2,15 @@ package com.scalableminds.webknossos.tracingstore.annotation import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.tools.Fox -import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing -import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeSegmentIndexBuffer +import com.scalableminds.util.tools.Fox.box2Fox +import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeTracingService import scala.concurrent.ExecutionContext trait AnnotationReversion { + def volumeTracingService: VolumeTracingService + def revertDistributedElements(annotationId: String, currentAnnotationWithTracings: AnnotationWithTracings, sourceAnnotationWithTracings: AnnotationWithTracings, @@ -18,8 +20,15 @@ trait AnnotationReversion { for { _ <- Fox.serialCombined(sourceAnnotationWithTracings.getVolumes) { // Only volume data for volume layers present in the *source annotation* needs to be reverted. - case (tracingId, sourceTracing) => Fox.successful(()) - //revertVolumeData(annotationId, tracingId, sourceTracing, revertAction.sourceVersion, newVersion) + case (tracingId, sourceTracing) => + for { + tracingBeforeRevert <- currentAnnotationWithTracings.getVolume(tracingId).toFox + _ <- volumeTracingService.revertVolumeData(tracingId, + revertAction.sourceVersion, + sourceTracing, + newVersion: Long, + tracingBeforeRevert) + } yield () } } yield () diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 567e9d47c0a..64b05098ecc 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -51,7 +51,7 @@ import scala.concurrent.ExecutionContext class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknossosClient, editableMappingService: EditableMappingService, - volumeTracingService: VolumeTracingService, + val volumeTracingService: VolumeTracingService, skeletonTracingService: SkeletonTracingService, val remoteDatastoreClient: TSRemoteDatastoreClient, tracingDataStore: TracingDataStore) @@ -122,18 +122,18 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss case a: EditableMappingUpdateAction => annotationWithTracings.applyEditableMappingAction(a) case a: RevertToVersionUpdateAction => - revertToVersion(annotationId, annotationWithTracings, a) + revertToVersion(annotationId, annotationWithTracings, a, targetVersion) // TODO if the revert action is not isolated, we need not the target version of all but the target version of this update case _: BucketMutatingVolumeUpdateAction => Fox.successful(annotationWithTracings) // No-op, as bucket-mutating actions are performed eagerly, so not here. case _ => Fox.failure(s"Received unsupported AnnotationUpdateAction action ${Json.toJson(updateAction)}") } } yield updated - private def revertToVersion(annotationId: String, - annotationWithTracings: AnnotationWithTracings, - revertAction: RevertToVersionUpdateAction)( - implicit ec: ExecutionContext, - tc: TokenContext): Fox[AnnotationWithTracings] = + private def revertToVersion( + annotationId: String, + annotationWithTracings: AnnotationWithTracings, + revertAction: RevertToVersionUpdateAction, + newVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = // Note: works only after “ironing out” the update action groups // TODO: read old annotationProto, tracing, buckets, segment indeces for { @@ -145,7 +145,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss requestAll = true) // TODO do we need to request the others? _ = logger.info( s"reverting to suorceVersion ${revertAction.sourceVersion}. got sourceAnnotation with version ${sourceAnnotation.version} with ${sourceAnnotation.skeletonStats}") - // _ <- revertDistributedElements(annotationId, annotationWithTracings, sourceAnnotation, revertAction) + _ <- revertDistributedElements(annotationId, annotationWithTracings, sourceAnnotation, revertAction, newVersion) } yield sourceAnnotation def createTracing(a: AddLayerAnnotationUpdateAction)( diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 1315db091ff..30b2df48631 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -250,19 +250,17 @@ class VolumeTracingService @Inject()( bool2Fox(mag.isIsotropic) } - /* - // TODO - private def revertToVolumeVersion(annotationId: String, - tracingId: String, - sourceVersion: Long, - newVersion: Long, - tracing: VolumeTracing)(implicit tc: TokenContext): Fox[VolumeTracing] = { + def revertVolumeData(tracingId: String, + sourceVersion: Long, + sourceTracing: VolumeTracing, + newVersion: Long, + tracing: VolumeTracing)(implicit tc: TokenContext): Fox[Unit] = { val dataLayer = volumeTracingLayer(tracingId, tracing) val bucketStream = dataLayer.volumeBucketProvider.bucketStreamWithVersion() for { - fallbackLayer <- getFallbackLayer(annotationId, tracingId) + fallbackLayer <- getFallbackLayer(tracingId, tracing) segmentIndexBuffer = new VolumeSegmentIndexBuffer(tracingId, volumeSegmentIndexClient, newVersion, @@ -270,8 +268,7 @@ class VolumeTracingService @Inject()( fallbackLayer, dataLayer.additionalAxes, tc) - sourceTracing <- find(annotationId, tracingId, Some(sourceVersion)) - mappingName <- baseMappingName(sourceTracing) + mappingName <- selectMappingName(sourceTracing) _ <- Fox.serialCombined(bucketStream) { case (bucketPosition, dataBeforeRevert, version) => if (version > sourceVersion) { @@ -310,9 +307,8 @@ class VolumeTracingService @Inject()( } else Fox.successful(()) } _ <- segmentIndexBuffer.flush() - } yield sourceTracing + } yield () } - */ def initializeWithDataMultiple(annotationId: String, tracingId: String, tracing: VolumeTracing, initialData: File)( implicit mp: MessagesProvider, From fe6257c601bb70b5b5c08bd97c42d5fcf2d0e1b1 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 9 Oct 2024 11:31:33 +0200 Subject: [PATCH 095/150] cache materialized with version --- .../tracingstore/TracingStoreModule.scala | 3 +- .../annotation/TSAnnotationService.scala | 50 +++++++++++++++---- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreModule.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreModule.scala index c0d36e2265c..49ac749ec1d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreModule.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TracingStoreModule.scala @@ -4,7 +4,7 @@ import org.apache.pekko.actor.ActorSystem import com.google.inject.AbstractModule import com.google.inject.name.Names import com.scalableminds.webknossos.datastore.services.AdHocMeshServiceHolder -import com.scalableminds.webknossos.tracingstore.annotation.AnnotationTransactionService +import com.scalableminds.webknossos.tracingstore.annotation.{AnnotationTransactionService, TSAnnotationService} import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService import com.scalableminds.webknossos.tracingstore.tracings.TracingDataStore import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService @@ -27,6 +27,7 @@ class TracingStoreModule extends AbstractModule { bind(classOf[TSSlackNotificationService]).asEagerSingleton() bind(classOf[AdHocMeshServiceHolder]).asEagerSingleton() bind(classOf[AnnotationTransactionService]).asEagerSingleton() + bind(classOf[TSAnnotationService]).asEagerSingleton() } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 64b05098ecc..bb43d1c6069 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -1,6 +1,7 @@ package com.scalableminds.webknossos.tracingstore.annotation import com.scalableminds.util.accesscontext.TokenContext +import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox import com.scalableminds.util.tools.Fox.{box2Fox, option2Fox} @@ -180,14 +181,43 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss withTracings <- getWithTracings(annotationId, version, List.empty, List.empty, requestAll = false) } yield withTracings.annotation - def getWithTracings( + private lazy val materializedAnnotationWithTracingCache = + // annotation id, version, requestedSkeletons, requestedVolumes, requestAll + // TODO instead of requested, use list of tracings determined from requests + updates? + AlfuCache[(String, Long, List[String], List[String], Boolean), AnnotationWithTracings]() + + private def getWithTracings( annotationId: String, version: Option[Long], requestedSkeletonTracingIds: List[String], requestedVolumeTracingIds: List[String], requestAll: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = for { - annotationWithVersion <- tracingDataStore.annotations.get(annotationId, version)(fromProtoBytes[AnnotationProto]) ?~> "getAnnotation.failed" + targetVersion <- determineTargetVersion(annotationId, version) ?~> "determineTargetVersion.failed" + updatedAnnotation <- materializedAnnotationWithTracingCache.getOrLoad( + (annotationId, targetVersion, requestedSkeletonTracingIds, requestedVolumeTracingIds, requestAll), + _ => + getWithTracingsVersioned( + annotationId, + targetVersion, + requestedSkeletonTracingIds, + requestedVolumeTracingIds, + requestAll = true) // TODO can we request fewer to save perf? still need to avoid duplicate apply + ) + } yield updatedAnnotation + + private def getWithTracingsVersioned( + annotationId: String, + version: Long, + requestedSkeletonTracingIds: List[String], + requestedVolumeTracingIds: List[String], + requestAll: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = + for { + annotationWithVersion <- tracingDataStore.annotations.get(annotationId, Some(version))( + fromProtoBytes[AnnotationProto]) ?~> "getAnnotation.failed" + _ = logger.info( + s"cache miss for ${annotationId} v$version, requested ${requestedSkeletonTracingIds.mkString(",")} + ${requestedVolumeTracingIds + .mkString(",")} (requestAll=$requestAll). Applying updates from ${annotationWithVersion.version} to $version...") annotation = annotationWithVersion.value updated <- applyPendingUpdates(annotation, annotationId, @@ -225,12 +255,11 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss private def applyPendingUpdates( annotation: AnnotationProto, annotationId: String, - targetVersionOpt: Option[Long], + targetVersion: Long, requestedSkeletonTracingIds: List[String], requestedVolumeTracingIds: List[String], requestAll: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = for { - targetVersion <- determineTargetVersion(annotation, annotationId, targetVersionOpt) ?~> "determineTargetVersion.failed" updates <- findPendingUpdates(annotationId, annotation.version, targetVersion) ?~> "findPendingUpdates.failed" annotationWithTracings <- findTracingsForUpdates(annotation, updates, @@ -371,7 +400,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss for { updated <- updateIter(Some(annotation), updates) updatedWithNewVerson = updated.withVersion(targetVersion) - _ = logger.info(s"flushing, with ${updated.skeletonStats}") + _ = logger.info(s"flushing v${targetVersion}, with ${updated.skeletonStats}") _ <- updatedWithNewVerson.flushBufferedUpdates() _ <- flushUpdatedTracings(updatedWithNewVerson) _ <- flushAnnotationInfo(annotationId, updatedWithNewVerson) @@ -406,18 +435,17 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss private def flushAnnotationInfo(annotationId: String, annotationWithTracings: AnnotationWithTracings) = tracingDataStore.annotations.put(annotationId, annotationWithTracings.version, annotationWithTracings.annotation) - private def determineTargetVersion(annotation: AnnotationProto, - annotationId: String, - targetVersionOpt: Option[Long]): Fox[Long] = + private def determineTargetVersion(annotationId: String, targetVersionOpt: Option[Long]): Fox[Long] = /* * Determines the newest saved version from the updates column. * if there are no updates at all, assume annotation is brand new (possibly created from NML, * hence the emptyFallbck annotation.version) */ for { - newestUpdateVersion <- tracingDataStore.annotationUpdates.getVersion(annotationId, - mayBeEmpty = Some(true), - emptyFallback = Some(annotation.version)) + newestUpdateVersion <- tracingDataStore.annotationUpdates.getVersion( + annotationId, + mayBeEmpty = Some(true), + emptyFallback = Some(0L)) // TODO in case of empty, look in annotation table, take version from there } yield { targetVersionOpt match { case None => newestUpdateVersion From 2d376402e27b2a04cd8e0ada9a4b1507bb07546d Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 14 Oct 2024 09:34:07 +0200 Subject: [PATCH 096/150] revert proofreading distributed elements --- .../tracingstore/annotation/AnnotationReversion.scala | 11 ++++++++++- .../annotation/AnnotationWithTracings.scala | 4 ++-- .../editablemapping/EditableMappingUpdater.scala | 10 ++-------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala index bb5bb384602..5e5deb0f035 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala @@ -2,7 +2,7 @@ package com.scalableminds.webknossos.tracingstore.annotation import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.tools.Fox -import com.scalableminds.util.tools.Fox.box2Fox +import com.scalableminds.util.tools.Fox.{box2Fox, option2Fox} import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeTracingService import scala.concurrent.ExecutionContext @@ -28,8 +28,17 @@ trait AnnotationReversion { sourceTracing, newVersion: Long, tracingBeforeRevert) + _ <- Fox.runIf(sourceTracing.getHasEditableMapping)( + revertEditableMappingFields(currentAnnotationWithTracings, revertAction, tracingId)) } yield () } } yield () + private def revertEditableMappingFields(currentAnnotationWithTracings: AnnotationWithTracings, + revertAction: RevertToVersionUpdateAction, + tracingId: String)(implicit ec: ExecutionContext): Fox[Unit] = + for { + updater <- currentAnnotationWithTracings.getEditableMappingUpdater(tracingId).toFox + _ <- updater.revertToVersion(revertAction) + } yield () } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index 42c41169f3d..a2094bfa70d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -71,7 +71,7 @@ case class AnnotationWithTracings( (info, _) <- editableMappingsByTracingId.get(tracingId) } yield info - private def getEditableMappingUpdater(tracingId: String): Option[EditableMappingUpdater] = + def getEditableMappingUpdater(tracingId: String): Option[EditableMappingUpdater] = for { (_, updater) <- editableMappingsByTracingId.get(tracingId) } yield updater @@ -130,7 +130,7 @@ case class AnnotationWithTracings( def applyEditableMappingAction(a: EditableMappingUpdateAction)( implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { - updater: EditableMappingUpdater <- getEditableMappingUpdater(a.actionTracingId).toFox // TODO editable mapping update actions need tracing id + updater: EditableMappingUpdater <- getEditableMappingUpdater(a.actionTracingId).toFox info <- getEditableMappingInfo(a.actionTracingId).toFox updated <- updater.applyOneUpdate(info, a) } yield diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index 715a3055984..09b2e04f031 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -128,8 +128,6 @@ class EditableMappingUpdater( applySplitAction(mapping, splitAction) ?~> "Failed to apply split action" case mergeAction: MergeAgglomerateUpdateAction => applyMergeAction(mapping, mergeAction) ?~> "Failed to apply merge action" - case revertAction: RevertToVersionUpdateAction => - revertToVersion(revertAction) ?~> "Failed to apply revert action" case _ => Fox.failure("this is not an editable mapping update action!") } @@ -420,13 +418,9 @@ class EditableMappingUpdater( ) } - private def revertToVersion(revertAction: RevertToVersionUpdateAction)( - implicit ec: ExecutionContext): Fox[EditableMappingInfo] = + def revertToVersion(revertAction: RevertToVersionUpdateAction)(implicit ec: ExecutionContext): Fox[Unit] = for { _ <- bool2Fox(revertAction.sourceVersion <= oldVersion) ?~> "trying to revert editable mapping to a version not yet present in the database" - oldInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId, Some(revertAction.sourceVersion))( - ec, - tokenContext) _ = segmentToAgglomerateBuffer.clear() _ = agglomerateToGraphBuffer.clear() segmentToAgglomerateChunkNewestStream = new VersionedSegmentToAgglomerateChunkIterator( @@ -466,6 +460,6 @@ class EditableMappingUpdater( } yield () } else Fox.successful(()) } - } yield oldInfo + } yield () } From 269e3500ea6182a67fe1ba3729cc491e7710d989 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 14 Oct 2024 09:45:52 +0200 Subject: [PATCH 097/150] fix volume revert logic --- .../annotation/AnnotationReversion.scala | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala index 5e5deb0f035..b85e053366d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala @@ -23,11 +23,12 @@ trait AnnotationReversion { case (tracingId, sourceTracing) => for { tracingBeforeRevert <- currentAnnotationWithTracings.getVolume(tracingId).toFox - _ <- volumeTracingService.revertVolumeData(tracingId, - revertAction.sourceVersion, - sourceTracing, - newVersion: Long, - tracingBeforeRevert) + _ <- Fox.runIf(!sourceTracing.getHasEditableMapping)( + volumeTracingService.revertVolumeData(tracingId, + revertAction.sourceVersion, + sourceTracing, + newVersion: Long, + tracingBeforeRevert)) _ <- Fox.runIf(sourceTracing.getHasEditableMapping)( revertEditableMappingFields(currentAnnotationWithTracings, revertAction, tracingId)) } yield () @@ -40,5 +41,6 @@ trait AnnotationReversion { for { updater <- currentAnnotationWithTracings.getEditableMappingUpdater(tracingId).toFox _ <- updater.revertToVersion(revertAction) + _ <- updater.flushBuffersToFossil() } yield () } From 50d9dc4a0e9fabb1cf06ce0c0e7690e4edfb87d2 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 14 Oct 2024 15:38:21 +0200 Subject: [PATCH 098/150] wip fix parallel revert --- .../annotation/TSAnnotationService.scala | 12 +++++------- .../volume/VolumeSegmentIndexService.scala | 4 +++- .../volume/VolumeTracingBucketHelper.scala | 2 +- .../tracings/volume/VolumeTracingService.scala | 17 ++++++++++------- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index bb43d1c6069..614f3b4a2cf 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -62,6 +62,11 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss with AnnotationReversion with LazyLogging { + private lazy val materializedAnnotationWithTracingCache = + // annotation id, version, requestedSkeletons, requestedVolumes, requestAll + // TODO instead of requested, use list of tracings determined from requests + updates? + AlfuCache[(String, Long, List[String], List[String], Boolean), AnnotationWithTracings](maxCapacity = 1) + def reportUpdates(annotationId: String, updateGroups: List[UpdateActionGroup])(implicit tc: TokenContext): Fox[Unit] = for { _ <- remoteWebknossosClient.reportTracingUpdates( @@ -181,11 +186,6 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss withTracings <- getWithTracings(annotationId, version, List.empty, List.empty, requestAll = false) } yield withTracings.annotation - private lazy val materializedAnnotationWithTracingCache = - // annotation id, version, requestedSkeletons, requestedVolumes, requestAll - // TODO instead of requested, use list of tracings determined from requests + updates? - AlfuCache[(String, Long, List[String], List[String], Boolean), AnnotationWithTracings]() - private def getWithTracings( annotationId: String, version: Option[Long], @@ -388,8 +388,6 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss remainingUpdates match { case List() => Fox.successful(annotationWithTracings) case update :: tail => - logger.info( - f"${remainingUpdates.length} remainingUpdates, current skeleton ${annotationWithTracings.skeletonStats})") updateIter(applyUpdate(annotationId, annotationWithTracings, update, targetVersion), tail) } case _ => annotationWithTracingsFox diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentIndexService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentIndexService.scala index a88c863f7ff..b04ce318747 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentIndexService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentIndexService.scala @@ -61,7 +61,9 @@ class VolumeSegmentIndexService @Inject()(val tracingDataStore: TracingDataStore editableMappingTracingId: Option[String])(implicit ec: ExecutionContext): Fox[Unit] = for { bucketBytesDecompressed <- tryo( - decompressIfNeeded(bucketBytes, expectedUncompressedBucketSizeFor(elementClass), "")).toFox + decompressIfNeeded(bucketBytes, + expectedUncompressedBucketSizeFor(elementClass), + "updating segment index, new bucket data")).toFox // previous bytes: include fallback layer bytes if available, otherwise use empty bytes previousBucketBytesWithEmptyFallback <- bytesWithEmptyFallback(previousBucketBytesBox, elementClass) ?~> "volumeSegmentIndex.update.getPreviousBucket.failed" segmentIds: Set[Long] <- collectSegmentIds(bucketBytesDecompressed, elementClass) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala index 0f3e8d06d5c..bec916c2bb5 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingBucketHelper.scala @@ -50,7 +50,7 @@ trait VolumeBucketCompression extends LazyLogging { } catch { case e: Exception => logger.error( - s"Failed to LZ4-decompress volume bucket ($debugInfo, expected uncompressed size $expectedUncompressedBucketSize): $e") + s"Failed to LZ4-decompress volume bucket ($debugInfo, compressed size: ${data.length}, expected uncompressed size $expectedUncompressedBucketSize): $e") throw e } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 30b2df48631..9c0ae6d7174 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -254,13 +254,16 @@ class VolumeTracingService @Inject()( sourceVersion: Long, sourceTracing: VolumeTracing, newVersion: Long, - tracing: VolumeTracing)(implicit tc: TokenContext): Fox[Unit] = { + tracingBeforeRevert: VolumeTracing)(implicit tc: TokenContext): Fox[Unit] = { - val dataLayer = volumeTracingLayer(tracingId, tracing) - val bucketStream = dataLayer.volumeBucketProvider.bucketStreamWithVersion() + val dataLayer = volumeTracingLayer(tracingId, tracingBeforeRevert) + val bucketStreamBeforeRevert = + dataLayer.volumeBucketProvider.bucketStreamWithVersion(version = Some(tracingBeforeRevert.version)) + + logger.info(s"reverting volume data from v${tracingBeforeRevert.version} to v$sourceVersion, creating v$newVersion") for { - fallbackLayer <- getFallbackLayer(tracingId, tracing) + fallbackLayer <- getFallbackLayer(tracingId, tracingBeforeRevert) segmentIndexBuffer = new VolumeSegmentIndexBuffer(tracingId, volumeSegmentIndexClient, newVersion, @@ -269,14 +272,14 @@ class VolumeTracingService @Inject()( dataLayer.additionalAxes, tc) mappingName <- selectMappingName(sourceTracing) - _ <- Fox.serialCombined(bucketStream) { + _ <- Fox.serialCombined(bucketStreamBeforeRevert) { case (bucketPosition, dataBeforeRevert, version) => if (version > sourceVersion) { loadBucket(dataLayer, bucketPosition, Some(sourceVersion)).futureBox.map { case Full(dataAfterRevert) => for { _ <- saveBucket(dataLayer, bucketPosition, dataAfterRevert, newVersion) - _ <- Fox.runIfOptionTrue(tracing.hasSegmentIndex)( + _ <- Fox.runIfOptionTrue(tracingBeforeRevert.hasSegmentIndex)( updateSegmentIndex( segmentIndexBuffer, bucketPosition, @@ -291,7 +294,7 @@ class VolumeTracingService @Inject()( for { dataAfterRevert <- Fox.successful(revertedValue) _ <- saveBucket(dataLayer, bucketPosition, dataAfterRevert, newVersion) - _ <- Fox.runIfOptionTrue(tracing.hasSegmentIndex)( + _ <- Fox.runIfOptionTrue(tracingBeforeRevert.hasSegmentIndex)( updateSegmentIndex( segmentIndexBuffer, bucketPosition, From e0e4dc4dd6abdaea306caf9f0b08291b6bdd993b Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 14 Oct 2024 15:47:52 +0200 Subject: [PATCH 099/150] in bucket data reversion, handle revertedValue --- .../annotation/TSAnnotationService.scala | 2 +- .../volume/VolumeSegmentIndexService.scala | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 614f3b4a2cf..9c55671815f 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -65,7 +65,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss private lazy val materializedAnnotationWithTracingCache = // annotation id, version, requestedSkeletons, requestedVolumes, requestAll // TODO instead of requested, use list of tracings determined from requests + updates? - AlfuCache[(String, Long, List[String], List[String], Boolean), AnnotationWithTracings](maxCapacity = 1) + AlfuCache[(String, Long, List[String], List[String], Boolean), AnnotationWithTracings](maxCapacity = 1000) def reportUpdates(annotationId: String, updateGroups: List[UpdateActionGroup])(implicit tc: TokenContext): Fox[Unit] = for { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentIndexService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentIndexService.scala index b04ce318747..4416f678d9a 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentIndexService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeSegmentIndexService.scala @@ -44,6 +44,7 @@ class VolumeSegmentIndexService @Inject()(val tracingDataStore: TracingDataStore with ProtoGeometryImplicits with VolumeBucketCompression with SegmentIndexKeyHelper + with ReversionHelper with LazyLogging { private val volumeSegmentIndexClient: FossilDBClient = tracingDataStore.volumeSegmentIndex @@ -60,10 +61,14 @@ class VolumeSegmentIndexService @Inject()(val tracingDataStore: TracingDataStore mappingName: Option[String], editableMappingTracingId: Option[String])(implicit ec: ExecutionContext): Fox[Unit] = for { - bucketBytesDecompressed <- tryo( - decompressIfNeeded(bucketBytes, - expectedUncompressedBucketSizeFor(elementClass), - "updating segment index, new bucket data")).toFox + bucketBytesDecompressed <- if (isRevertedElement(bucketBytes)) { + Fox.successful(emptyArrayForElementClass(elementClass)) + } else { + tryo( + decompressIfNeeded(bucketBytes, + expectedUncompressedBucketSizeFor(elementClass), + "updating segment index, new bucket data")).toFox + } // previous bytes: include fallback layer bytes if available, otherwise use empty bytes previousBucketBytesWithEmptyFallback <- bytesWithEmptyFallback(previousBucketBytesBox, elementClass) ?~> "volumeSegmentIndex.update.getPreviousBucket.failed" segmentIds: Set[Long] <- collectSegmentIds(bucketBytesDecompressed, elementClass) @@ -90,11 +95,14 @@ class VolumeSegmentIndexService @Inject()(val tracingDataStore: TracingDataStore private def bytesWithEmptyFallback(bytesBox: Box[Array[Byte]], elementClass: ElementClassProto)( implicit ec: ExecutionContext): Fox[Array[Byte]] = bytesBox match { - case Empty => Fox.successful(Array.fill[Byte](ElementClass.bytesPerElement(elementClass))(0)) + case Empty => Fox.successful(emptyArrayForElementClass(elementClass)) case Full(bytes) => Fox.successful(bytes) case f: Failure => f.toFox } + private def emptyArrayForElementClass(elementClass: ElementClassProto): Array[Byte] = + Array.fill[Byte](ElementClass.bytesPerElement(elementClass))(0) + private def removeBucketFromSegmentIndex( segmentIndexBuffer: VolumeSegmentIndexBuffer, segmentId: Long, From 8ea36da96799ca60047bc756ecf26ce4e119beef Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 15 Oct 2024 09:45:39 +0200 Subject: [PATCH 100/150] split and isolate function --- .../scala/collections/SequenceUtils.scala | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/util/src/main/scala/collections/SequenceUtils.scala b/util/src/main/scala/collections/SequenceUtils.scala index 9d839141c7d..d047e352c3f 100644 --- a/util/src/main/scala/collections/SequenceUtils.scala +++ b/util/src/main/scala/collections/SequenceUtils.scala @@ -1,9 +1,40 @@ package collections +import scala.annotation.tailrec + object SequenceUtils { def findUniqueElement[T](list: Seq[T]): Option[T] = { val uniqueElements = list.distinct if (uniqueElements.length == 1) uniqueElements.headOption else None } + + /* + Split a list into n parts, isolating the elements that satisfy the given predicate. + Those elements will be in single-item lists + Example: + splitAndIsolate(List(1,2,3,4,5,6,7))(i => i == 4) + → List(List(1, 2, 3), List(4), List(5, 6, 7)) + splitAndIsolate(List(1,2,3,4,5,6,7))(i => i % 3 == 0) + → List(List(1, 2), List(3), List(4, 5), List(6), List(7)) + splitAndIsolate(List(1,2,3,4,5,6,7))(i => i > 1000) # no matches → no splitting + → List(List(1, 2, 3, 4, 5, 6, 7)) + splitAndIsolate(List())(i => true) # empty list stays empty + → List() + */ + def splitAndIsolate[T](list: List[T])(predicate: T => Boolean): List[List[T]] = + list + .foldLeft(List[List[T]]()) { (acc, item) => + if (predicate(item)) { + List.empty :: List(item) :: acc + } else { + acc match { + case head :: tail => (item :: head) :: tail + case Nil => List(List(item)) + } + } + } + .reverse // we prepended on the outer list (for perf reasons) + .map(_.reverse) // we prepended on the inner lists (for perf reasons) + } From 5bdce21a98d54d6208b80624a74898a4ffe887d0 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 15 Oct 2024 10:12:39 +0200 Subject: [PATCH 101/150] WIP: Unified Annotation Versioning: regroup update groups by revert actions --- .../scala/collections/SequenceUtils.scala | 2 - .../annotation/TSAnnotationService.scala | 54 +++++++++++++------ 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/util/src/main/scala/collections/SequenceUtils.scala b/util/src/main/scala/collections/SequenceUtils.scala index d047e352c3f..a584ae2923d 100644 --- a/util/src/main/scala/collections/SequenceUtils.scala +++ b/util/src/main/scala/collections/SequenceUtils.scala @@ -1,7 +1,5 @@ package collections -import scala.annotation.tailrec - object SequenceUtils { def findUniqueElement[T](list: Seq[T]): Option[T] = { val uniqueElements = list.distinct diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 9c55671815f..5c47e2324c4 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -1,5 +1,6 @@ package com.scalableminds.webknossos.tracingstore.annotation +import collections.SequenceUtils import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.time.Instant @@ -87,7 +88,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss tracingDataStore.annotations.getVersion(annotationId, mayBeEmpty = Some(true), emptyFallback = Some(0L)) private def findPendingUpdates(annotationId: String, existingVersion: Long, desiredVersion: Long)( - implicit ec: ExecutionContext): Fox[List[UpdateAction]] = + implicit ec: ExecutionContext): Fox[List[(Long, List[UpdateAction])]] = if (desiredVersion == existingVersion) Fox.successful(List()) else { for { @@ -95,8 +96,8 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss annotationId, Some(desiredVersion), Some(existingVersion + 1))(fromJsonBytes[List[UpdateAction]]) - updateActionGroupsWithVersionsIroned = ironOutReversionFolds(updateActionGroupsWithVersions) - } yield updateActionGroupsWithVersionsIroned + updateActionGroupsWithVersionsRegrouped = regroupByRevertActions(updateActionGroupsWithVersions) + } yield updateActionGroupsWithVersionsRegrouped } // TODO option to dry apply? @@ -260,18 +261,20 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss requestedVolumeTracingIds: List[String], requestAll: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = for { - updates <- findPendingUpdates(annotationId, annotation.version, targetVersion) ?~> "findPendingUpdates.failed" + updatesRegrouped <- findPendingUpdates(annotationId, annotation.version, targetVersion) ?~> "findPendingUpdates.failed" + updatesFlat = updatesRegrouped.flatMap(_._2) annotationWithTracings <- findTracingsForUpdates(annotation, - updates, + updatesFlat, requestedSkeletonTracingIds, requestedVolumeTracingIds, requestAll) ?~> "findTracingsForUpdates.failed" - annotationWithTracingsAndMappings <- findEditableMappingsForUpdates(annotationId, - annotationWithTracings, - updates, - annotation.version, - targetVersion) - updated <- applyUpdates(annotationWithTracingsAndMappings, annotationId, updates, targetVersion) ?~> "applyUpdates.inner.failed" + annotationWithTracingsAndMappings <- findEditableMappingsForUpdates( + annotationId, + annotationWithTracings, + updatesFlat, + annotation.version, + targetVersion) // TODO: targetVersion should be set per update group + updated <- applyUpdatesGrouped(annotationWithTracingsAndMappings, annotationId, updatesRegrouped) ?~> "applyUpdates.inner.failed" } yield updated private def findEditableMappingsForUpdates( // TODO integrate with findTracings? @@ -374,6 +377,12 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } yield AnnotationWithTracings(annotation, skeletonTracingsMap ++ volumeTracingsMap, Map.empty) } + private def applyUpdatesGrouped( + annotation: AnnotationWithTracings, + annotationId: String, + updates: List[(Long, List[UpdateAction])] + ): Fox[AnnotationWithTracings] = ??? + private def applyUpdates( annotation: AnnotationWithTracings, annotationId: String, @@ -406,10 +415,25 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } } - private def ironOutReversionFolds( - updateActionGroupsWithVersions: List[(Long, List[UpdateAction])]): List[UpdateAction] = - // TODO: if the source version is in the current update list, it needs to be ironed out. in case of overlaps, iron out from the back. - updateActionGroupsWithVersions.reverse.flatMap(_._2) + private def regroupByRevertActions( + updateActionGroupsWithVersions: List[(Long, List[UpdateAction])]): List[(Long, List[UpdateAction])] = { + val splitGroupLists: List[List[(Long, List[UpdateAction])]] = + SequenceUtils.splitAndIsolate(updateActionGroupsWithVersions.reverse)(actionGroup => + actionGroup._2.contains(updateAction => isRevertAction(updateAction))) + // TODO assert that the groups that contain revert actions contain nothing else + // TODO test this + + splitGroupLists.flatMap { groupsToConcatenate: List[(Long, List[UpdateAction])] => + val updates = groupsToConcatenate.flatMap(_._2) + val targetVersionOpt: Option[Long] = groupsToConcatenate.map(_._1).headOption + targetVersionOpt.map(targetVersion => (targetVersion, updates)) + } + } + + private def isRevertAction(a: UpdateAction): Boolean = a match { + case _: RevertToVersionUpdateAction => true + case _ => false + } private def flushUpdatedTracings(annotationWithTracings: AnnotationWithTracings)(implicit ec: ExecutionContext) = // TODO skip some flushes to save disk space (e.g. skeletons only nth version, or only if requested?) From 3f48feae28cd635482364b2d883a9892b2c6ead6 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 15 Oct 2024 10:22:34 +0200 Subject: [PATCH 102/150] wip apply regrouped updates --- .../annotation/AnnotationWithTracings.scala | 7 ++++++ .../annotation/TSAnnotationService.scala | 22 ++++++++++++++++--- .../EditableMappingUpdater.scala | 1 + 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index a2094bfa70d..5b02bb451b8 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -106,6 +106,13 @@ case class AnnotationWithTracings( this.copy(annotation = annotation.copy(version = newVersion), tracingsById = tracingsUpdated.toMap) } + def withTargetVersion(targetVersion: Long): AnnotationWithTracings = { + val editableMappingsUpdated = editableMappingsByTracingId.view.mapValues { + case (mapping, updater) => (mapping, updater.withTargetVersion(targetVersion)) + } + this.copy(editableMappingsByTracingId = editableMappingsUpdated.toMap) + } + def addEditableMapping(volumeTracingId: String, editableMappingInfo: EditableMappingInfo, updater: EditableMappingUpdater): AnnotationWithTracings = diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 5c47e2324c4..4f7f32dc975 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -380,8 +380,24 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss private def applyUpdatesGrouped( annotation: AnnotationWithTracings, annotationId: String, - updates: List[(Long, List[UpdateAction])] - ): Fox[AnnotationWithTracings] = ??? + updateGroups: List[(Long, List[UpdateAction])] + )(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = { + def updateGroupedIter(annotationWithTracingsFox: Fox[AnnotationWithTracings], + remainingUpdateGroups: List[(Long, List[UpdateAction])]): Fox[AnnotationWithTracings] = + annotationWithTracingsFox.futureBox.flatMap { + case Empty => Fox.empty + case Full(annotationWithTracings) => + remainingUpdateGroups match { + case List() => Fox.successful(annotationWithTracings) + case updateGroup :: tail => + updateGroupedIter(applyUpdates(annotationWithTracings, annotationId, updateGroup._2, updateGroup._1), + tail) + } + case _ => annotationWithTracingsFox + } + + updateGroupedIter(Some(annotation), updateGroups) + } private def applyUpdates( annotation: AnnotationWithTracings, @@ -405,7 +421,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss if (updates.isEmpty) Full(annotation) else { for { - updated <- updateIter(Some(annotation), updates) + updated <- updateIter(Some(annotation.withTargetVersion(targetVersion)), updates) updatedWithNewVerson = updated.withVersion(targetVersion) _ = logger.info(s"flushing v${targetVersion}, with ${updated.skeletonStats}") _ <- updatedWithNewVerson.flushBufferedUpdates() diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index 09b2e04f031..4acaa219889 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -462,4 +462,5 @@ class EditableMappingUpdater( } } yield () + def withTargetVersion(targetVersion: Long): EditableMappingUpdater = ??? // TODO build new or copy? } From e98510d911048dbb8fd619fc5c0148ee945e26d3 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 15 Oct 2024 10:28:04 +0200 Subject: [PATCH 103/150] build new updater --- .../annotation/AnnotationWithTracings.scala | 4 ++-- .../annotation/TSAnnotationService.scala | 2 +- .../editablemapping/EditableMappingUpdater.scala | 16 +++++++++++++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index 5b02bb451b8..658cac9da17 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -106,9 +106,9 @@ case class AnnotationWithTracings( this.copy(annotation = annotation.copy(version = newVersion), tracingsById = tracingsUpdated.toMap) } - def withTargetVersion(targetVersion: Long): AnnotationWithTracings = { + def withNewUpdaters(materializedVersion: Long, targetVersion: Long): AnnotationWithTracings = { val editableMappingsUpdated = editableMappingsByTracingId.view.mapValues { - case (mapping, updater) => (mapping, updater.withTargetVersion(targetVersion)) + case (mapping, updater) => (mapping, updater.newWithTargetVersion(materializedVersion, targetVersion)) } this.copy(editableMappingsByTracingId = editableMappingsUpdated.toMap) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 4f7f32dc975..31d604f8fc3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -421,7 +421,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss if (updates.isEmpty) Full(annotation) else { for { - updated <- updateIter(Some(annotation.withTargetVersion(targetVersion)), updates) + updated <- updateIter(Some(annotation.withNewUpdaters(annotation.version, targetVersion)), updates) updatedWithNewVerson = updated.withVersion(targetVersion) _ = logger.info(s"flushing v${targetVersion}, with ${updated.skeletonStats}") _ <- updatedWithNewVerson.flushBufferedUpdates() diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index 4acaa219889..15f5fda72cc 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -462,5 +462,19 @@ class EditableMappingUpdater( } } yield () - def withTargetVersion(targetVersion: Long): EditableMappingUpdater = ??? // TODO build new or copy? + def newWithTargetVersion(currentMaterializedVersion: Long, targetVersion: Long): EditableMappingUpdater = + new EditableMappingUpdater( + annotationId, + tracingId, + baseMappingName, + currentMaterializedVersion, + targetVersion, + remoteFallbackLayer, + tokenContext, + remoteDatastoreClient, + editableMappingService, + annotationService, + tracingDataStore, + relyOnAgglomerateIds = relyOnAgglomerateIds + ) } From e098113e5e3b91bca1e2fff2c987ab5822b93274 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 15 Oct 2024 12:04:04 +0200 Subject: [PATCH 104/150] fix regroup --- .../UpdateGroupHandlingUnitTestSuite.scala | 39 +++++++++++++++++ .../AnnotationTransactionService.scala | 2 +- .../annotation/TSAnnotationService.scala | 43 ++++++------------- .../annotation/UpdateGroupHandling.scala | 31 +++++++++++++ .../tracings/TracingDataStore.scala | 1 + 5 files changed, 84 insertions(+), 32 deletions(-) create mode 100644 test/backend/UpdateGroupHandlingUnitTestSuite.scala create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateGroupHandling.scala diff --git a/test/backend/UpdateGroupHandlingUnitTestSuite.scala b/test/backend/UpdateGroupHandlingUnitTestSuite.scala new file mode 100644 index 00000000000..bdc06c79f36 --- /dev/null +++ b/test/backend/UpdateGroupHandlingUnitTestSuite.scala @@ -0,0 +1,39 @@ +package backend + +import com.scalableminds.webknossos.tracingstore.annotation.{RevertToVersionUpdateAction, UpdateGroupHandling} +import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{MergeTreeSkeletonAction} +import org.scalatestplus.play.PlaySpec + +class UpdateGroupHandlingUnitTestSuite extends PlaySpec with UpdateGroupHandling { + + "regroup" should { + "work" in { + val updateGroupsBefore = List( + (5L, + List( + MergeTreeSkeletonAction(sourceId = 1, targetId = 2, actionTracingId = Dummies.tracingId), + MergeTreeSkeletonAction(sourceId = 2, targetId = 3, actionTracingId = Dummies.tracingId) + )), + (6L, + List( + RevertToVersionUpdateAction(sourceVersion = 1), + )), + (7L, + List( + MergeTreeSkeletonAction(sourceId = 1, targetId = 2, actionTracingId = Dummies.tracingId), + MergeTreeSkeletonAction(sourceId = 2, targetId = 3, actionTracingId = Dummies.tracingId) + )), + (8L, + List( + MergeTreeSkeletonAction(sourceId = 1, targetId = 2, actionTracingId = Dummies.tracingId), + MergeTreeSkeletonAction(sourceId = 2, targetId = 3, actionTracingId = Dummies.tracingId) + )), + ) + val res = regroupByRevertActions(updateGroupsBefore) + assert(res.length == 3) + assert(res(1)._2.length == 1) + assert(res(1)._1 == 6L) + } + } + +} diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala index e96ffd31641..33e81fcfe85 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala @@ -165,7 +165,7 @@ class AnnotationTransactionService @Inject()(handledGroupIdStore: TracingStoreRe currentCommittedVersion: Fox[Long] = annotationService.currentMaterializableVersion(annotationId) _ = logger.info(s"trying to commit ${updateGroups .map(_.actions.length) - .sum} actions in ${updateGroups.length} groups (versions ${updateGroups.map(_.version).mkString(",")}") + .sum} actions in ${updateGroups.length} groups (versions ${updateGroups.map(_.version).mkString(",")})") newVersion <- updateGroups.foldLeft(currentCommittedVersion) { (previousVersion, updateGroup) => previousVersion.flatMap { prevVersion: Long => if (prevVersion + 1 == updateGroup.version) { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 31d604f8fc3..fff6b7890bf 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -1,6 +1,5 @@ package com.scalableminds.webknossos.tracingstore.annotation -import collections.SequenceUtils import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.time.Instant @@ -61,6 +60,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss with FallbackDataHelper with ProtoGeometryImplicits with AnnotationReversion + with UpdateGroupHandling with LazyLogging { private lazy val materializedAnnotationWithTracingCache = @@ -91,13 +91,10 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss implicit ec: ExecutionContext): Fox[List[(Long, List[UpdateAction])]] = if (desiredVersion == existingVersion) Fox.successful(List()) else { - for { - updateActionGroupsWithVersions <- tracingDataStore.annotationUpdates.getMultipleVersionsAsVersionValueTuple( - annotationId, - Some(desiredVersion), - Some(existingVersion + 1))(fromJsonBytes[List[UpdateAction]]) - updateActionGroupsWithVersionsRegrouped = regroupByRevertActions(updateActionGroupsWithVersions) - } yield updateActionGroupsWithVersionsRegrouped + tracingDataStore.annotationUpdates.getMultipleVersionsAsVersionValueTuple( + annotationId, + Some(desiredVersion), + Some(existingVersion + 1))(fromJsonBytes[List[UpdateAction]]) } // TODO option to dry apply? @@ -261,8 +258,9 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss requestedVolumeTracingIds: List[String], requestAll: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = for { - updatesRegrouped <- findPendingUpdates(annotationId, annotation.version, targetVersion) ?~> "findPendingUpdates.failed" - updatesFlat = updatesRegrouped.flatMap(_._2) + updateGroupsAsSaved <- findPendingUpdates(annotationId, annotation.version, targetVersion) ?~> "findPendingUpdates.failed" + updatesGroupsRegrouped = regroupByRevertActions(updateGroupsAsSaved) + updatesFlat = updatesGroupsRegrouped.flatMap(_._2) annotationWithTracings <- findTracingsForUpdates(annotation, updatesFlat, requestedSkeletonTracingIds, @@ -274,7 +272,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss updatesFlat, annotation.version, targetVersion) // TODO: targetVersion should be set per update group - updated <- applyUpdatesGrouped(annotationWithTracingsAndMappings, annotationId, updatesRegrouped) ?~> "applyUpdates.inner.failed" + updated <- applyUpdatesGrouped(annotationWithTracingsAndMappings, annotationId, updatesGroupsRegrouped) ?~> "applyUpdates.inner.failed" } yield updated private def findEditableMappingsForUpdates( // TODO integrate with findTracings? @@ -405,6 +403,9 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss updates: List[UpdateAction], targetVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = { + logger.info(s"applying ${updates.length} to go from v${annotation.version} to v$targetVersion") + + // TODO can we make this tail recursive? def updateIter(annotationWithTracingsFox: Fox[AnnotationWithTracings], remainingUpdates: List[UpdateAction]): Fox[AnnotationWithTracings] = annotationWithTracingsFox.futureBox.flatMap { @@ -431,26 +432,6 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } } - private def regroupByRevertActions( - updateActionGroupsWithVersions: List[(Long, List[UpdateAction])]): List[(Long, List[UpdateAction])] = { - val splitGroupLists: List[List[(Long, List[UpdateAction])]] = - SequenceUtils.splitAndIsolate(updateActionGroupsWithVersions.reverse)(actionGroup => - actionGroup._2.contains(updateAction => isRevertAction(updateAction))) - // TODO assert that the groups that contain revert actions contain nothing else - // TODO test this - - splitGroupLists.flatMap { groupsToConcatenate: List[(Long, List[UpdateAction])] => - val updates = groupsToConcatenate.flatMap(_._2) - val targetVersionOpt: Option[Long] = groupsToConcatenate.map(_._1).headOption - targetVersionOpt.map(targetVersion => (targetVersion, updates)) - } - } - - private def isRevertAction(a: UpdateAction): Boolean = a match { - case _: RevertToVersionUpdateAction => true - case _ => false - } - private def flushUpdatedTracings(annotationWithTracings: AnnotationWithTracings)(implicit ec: ExecutionContext) = // TODO skip some flushes to save disk space (e.g. skeletons only nth version, or only if requested?) for { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateGroupHandling.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateGroupHandling.scala new file mode 100644 index 00000000000..6b8f4cf0e04 --- /dev/null +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateGroupHandling.scala @@ -0,0 +1,31 @@ +package com.scalableminds.webknossos.tracingstore.annotation + +import collections.SequenceUtils + +trait UpdateGroupHandling { + + def regroupByRevertActions( + updateActionGroupsWithVersions: List[(Long, List[UpdateAction])]): List[(Long, List[UpdateAction])] = { + val splitGroupLists: List[List[(Long, List[UpdateAction])]] = + SequenceUtils.splitAndIsolate(updateActionGroupsWithVersions.reverse)(actionGroup => + actionGroup._2.exists(updateAction => isRevertAction(updateAction))) + // TODO assert that the groups that contain revert actions contain nothing else + // TODO test this + + splitGroupLists.flatMap { groupsToConcatenate: List[(Long, List[UpdateAction])] => + concatenateUpdateActionGroups(groupsToConcatenate) + } + } + + private def concatenateUpdateActionGroups( + groups: List[(Long, List[UpdateAction])]): Option[(Long, List[UpdateAction])] = { + val updates = groups.flatMap(_._2) + val targetVersionOpt: Option[Long] = groups.map(_._1).lastOption + targetVersionOpt.map(targetVersion => (targetVersion, updates)) + } + + private def isRevertAction(a: UpdateAction): Boolean = a match { + case _: RevertToVersionUpdateAction => true + case _ => false + } +} diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala index e5e5d0ad777..76be451e007 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingDataStore.scala @@ -44,6 +44,7 @@ class TracingDataStore @Inject()(config: TracingStoreConfig, healthClient.shutdown() skeletons.shutdown() annotationUpdates.shutdown() + annotations.shutdown() volumes.shutdown() volumeData.shutdown() editableMappingsInfo.shutdown() From d5f1a62da96ef23019b0246c79cec782875b42a8 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 16 Oct 2024 13:36:37 +0200 Subject: [PATCH 105/150] delete annotation layer via update action --- .../WKRemoteTracingStoreController.scala | 23 +++++++++- app/models/analytics/AnalyticsService.scala | 1 + app/models/annotation/Annotation.scala | 9 +++- app/models/annotation/AnnotationService.scala | 4 +- conf/webknossos.latest.routes | 1 + .../models/annotation/AnnotationLayer.scala | 42 ++++++++++--------- .../annotation/AnnotationLayerType.scala | 6 +++ .../webknossos/datastore/rpc/RPCRequest.scala | 4 +- .../slacknotification/SlackClient.scala | 1 + .../TSRemoteWebknossosClient.scala | 7 ++++ .../AnnotationTransactionService.scala | 13 ++++++ .../annotation/AnnotationUpdateActions.scala | 4 +- .../annotation/TSAnnotationService.scala | 5 +++ .../annotation/UpdateActions.scala | 2 + 14 files changed, 95 insertions(+), 27 deletions(-) diff --git a/app/controllers/WKRemoteTracingStoreController.scala b/app/controllers/WKRemoteTracingStoreController.scala index f3b488b8060..14eb8e2146b 100644 --- a/app/controllers/WKRemoteTracingStoreController.scala +++ b/app/controllers/WKRemoteTracingStoreController.scala @@ -3,6 +3,7 @@ package controllers import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext} import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.{Fox, FoxImplicits} +import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayer import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId import com.scalableminds.webknossos.tracingstore.TracingUpdatesReport import com.scalableminds.webknossos.tracingstore.tracings.TracingIds @@ -51,6 +52,26 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore val bearerTokenService: WebknossosBearerTokenAuthenticatorService = wkSilhouetteEnvironment.combinedAuthenticatorService.tokenAuthenticatorService + def updateAnnotationLayers(name: String, key: String, annotationId: String): Action[List[AnnotationLayer]] = + Action.async(validateJson[List[AnnotationLayer]]) { implicit request => + for { + annotationIdValidated <- ObjectId.fromString(annotationId) + existingLayers <- annotationLayerDAO.findAnnotationLayersFor(annotationIdValidated) + existingLayerIds = existingLayers.map(_.tracingId).toSet + newLayerIds = request.body.map(_.tracingId).toSet + layerIdsToDelete = existingLayerIds.diff(newLayerIds) + layerIdsToUpdate = existingLayerIds.intersect(newLayerIds) + layerIdsToInsert = newLayerIds.diff(existingLayerIds) + _ <- Fox.serialCombined(layerIdsToDelete.toList)( + annotationLayerDAO.deleteOneByTracingId(annotationIdValidated, _)) + _ <- Fox.serialCombined(request.body.filter(l => layerIdsToInsert.contains(l.tracingId)))( + annotationLayerDAO.insertOne(annotationIdValidated, _)) + _ <- Fox.serialCombined(request.body.filter(l => layerIdsToUpdate.contains(l.tracingId)))(l => + annotationLayerDAO.updateName(annotationIdValidated, l.tracingId, l.name)) + // Layer stats are ignored here, they are sent eagerly when saving updates + } yield Ok + } + def handleTracingUpdateReport(name: String, key: String): Action[TracingUpdatesReport] = Action.async(validateJson[TracingUpdatesReport]) { implicit request => implicit val ctx: DBAccessContext = GlobalAccessContext @@ -63,7 +84,7 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore _ <- annotationDAO.updateModified(annotation._id, Instant.now) /*_ <- Fox.runOptional(report.statistics) { statistics => annotationLayerDAO.updateStatistics(annotation._id, annotationId, statistics) - }*/ // TODO stats per tracing id + }*/ // TODO stats per tracing id. note: they might arrive before the layer is created. skip them then. userBox <- bearerTokenService.userForTokenOpt(report.userToken).futureBox trackTime = report.significantChangesCount > 0 || !wkConf.WebKnossos.User.timeTrackingOnlyWithSignificantChanges _ <- Fox.runOptional(userBox)(user => diff --git a/app/models/analytics/AnalyticsService.scala b/app/models/analytics/AnalyticsService.scala index 86b2cec45c1..83aba641736 100644 --- a/app/models/analytics/AnalyticsService.scala +++ b/app/models/analytics/AnalyticsService.scala @@ -55,6 +55,7 @@ class AnalyticsService @Inject()(rpc: RPC, } val wrappedJson = Json.obj("api_key" -> conf.key, "events" -> List(analyticsEventJson)) rpc(conf.uri).silent.postJson(wrappedJson) + () } Fox.successful(()) } diff --git a/app/models/annotation/Annotation.scala b/app/models/annotation/Annotation.scala index 887bf7439c9..493bce4bdd4 100755 --- a/app/models/annotation/Annotation.scala +++ b/app/models/annotation/Annotation.scala @@ -140,13 +140,20 @@ class AnnotationLayerDAO @Inject()(SQLClient: SqlClient)(implicit ec: ExecutionC q"""INSERT INTO webknossos.annotation_layers(_annotation, tracingId, typ, name, statistics) VALUES($annotationId, ${a.tracingId}, ${a.typ}, ${a.name}, ${a.stats})""".asUpdate - def deleteOne(annotationId: ObjectId, layerName: String): Fox[Unit] = + def deleteOneByName(annotationId: ObjectId, layerName: String): Fox[Unit] = for { _ <- run(q"""DELETE FROM webknossos.annotation_layers WHERE _annotation = $annotationId AND name = $layerName""".asUpdate) } yield () + def deleteOneByTracingId(annotationId: ObjectId, tracingId: String): Fox[Unit] = + for { + _ <- run(q"""DELETE FROM webknossos.annotation_layers + WHERE _annotation = $annotationId + AND tracingId = $tracingId""".asUpdate) + } yield () + def findAnnotationIdByTracingId(tracingId: String): Fox[ObjectId] = for { rList <- run(q"SELECT _annotation FROM webknossos.annotation_layers WHERE tracingId = $tracingId".as[ObjectId]) diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index 26dffa21ad7..57327d2924e 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -215,7 +215,7 @@ class AnnotationService @Inject()( def deleteAnnotationLayer(annotation: Annotation, layerName: String): Fox[Unit] = for { - _ <- annotationLayersDAO.deleteOne(annotation._id, layerName) + _ <- annotationLayersDAO.deleteOneByName(annotation._id, layerName) } yield () private def createTracingsForExplorational(dataset: Dataset, @@ -314,7 +314,7 @@ class AnnotationService @Inject()( AnnotationLayer(tracingIdAndName._1, annotationLayerParameters.typ, tracingIdAndName._2, - AnnotationLayerStatistics.zeroedForTyp(annotationLayerParameters.typ)) + AnnotationLayerStatistics.zeroedForType(annotationLayerParameters.typ)) def fetchOldPrecedenceLayer: Fox[Option[FetchedAnnotationLayer]] = if (existingAnnotationLayers.isEmpty) Fox.successful(None) diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes index 5b9ad059207..64fdcd470a4 100644 --- a/conf/webknossos.latest.routes +++ b/conf/webknossos.latest.routes @@ -121,6 +121,7 @@ PUT /datastores/:name # Tracingstores GET /tracingstore controllers.TracingStoreController.listOne() POST /tracingstores/:name/handleTracingUpdateReport controllers.WKRemoteTracingStoreController.handleTracingUpdateReport(name: String, key: String) +POST /tracingstores/:name/updateAnnotationLayers controllers.WKRemoteTracingStoreController.updateAnnotationLayers(name: String, key: String, annotationId: String) POST /tracingstores/:name/validateUserAccess controllers.UserTokenController.validateAccessViaTracingstore(name: String, key: String, token: Option[String]) PUT /tracingstores/:name controllers.TracingStoreController.update(name: String) GET /tracingstores/:name/dataSource controllers.WKRemoteTracingStoreController.dataSourceForTracing(name: String, key: String, tracingId: String) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayer.scala index a6d0c65c8c5..1c8e827d31c 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayer.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayer.scala @@ -2,6 +2,7 @@ package com.scalableminds.webknossos.datastore.models.annotation import com.scalableminds.util.tools.Fox.bool2Fox import com.scalableminds.util.tools.{Fox, FoxImplicits} +import com.scalableminds.webknossos.datastore.Annotation.AnnotationLayerProto import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType.AnnotationLayerType @@ -17,28 +18,12 @@ case class AnnotationLayer( stats: JsObject, ) -object AnnotationLayerStatistics { - - def zeroedForTyp(typ: AnnotationLayerType): JsObject = typ match { - case AnnotationLayerType.Skeleton => - Json.obj( - "treeCount" -> 0, - "nodeCount" -> 0, - "edgeCount" -> 0, - "branchPointCount" -> 0 - ) - case AnnotationLayerType.Volume => - Json.obj( - "segmentCount" -> 0 - ) - } - - def unknown: JsObject = Json.obj() -} - object AnnotationLayer extends FoxImplicits { implicit val jsonFormat: OFormat[AnnotationLayer] = Json.format[AnnotationLayer] + def fromProto(p: AnnotationLayerProto): AnnotationLayer = + AnnotationLayer(p.tracingId, AnnotationLayerType.fromProto(p.`type`), p.name, AnnotationLayerStatistics.unknown) + val defaultSkeletonLayerName: String = "Skeleton" val defaultVolumeLayerName: String = "Volume" @@ -63,6 +48,25 @@ object AnnotationLayer extends FoxImplicits { } } +object AnnotationLayerStatistics { + + def zeroedForType(typ: AnnotationLayerType): JsObject = typ match { + case AnnotationLayerType.Skeleton => + Json.obj( + "treeCount" -> 0, + "nodeCount" -> 0, + "edgeCount" -> 0, + "branchPointCount" -> 0 + ) + case AnnotationLayerType.Volume => + Json.obj( + "segmentCount" -> 0 + ) + } + + def unknown: JsObject = Json.obj() +} + case class FetchedAnnotationLayer(tracingId: String, name: String, tracing: Either[SkeletonTracing, VolumeTracing], diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayerType.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayerType.scala index 756180cbbd8..2f34d15b159 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayerType.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayerType.scala @@ -12,4 +12,10 @@ object AnnotationLayerType extends ExtendedEnumeration { case Skeleton => AnnotationLayerTypeProto.skeleton case Volume => AnnotationLayerTypeProto.volume } + + def fromProto(p: AnnotationLayerTypeProto): AnnotationLayerType = + p match { + case AnnotationLayerTypeProto.skeleton => Skeleton + case AnnotationLayerTypeProto.volume => Volume + } } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala index 4bd9e2872eb..ba8aee37c6d 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala @@ -166,10 +166,10 @@ class RPCRequest(val id: Int, val url: String, wsClient: WSClient)(implicit ec: parseProtoResponse(performRequest)(companion) } - def postJson[J: Writes](body: J = Json.obj()): Unit = { + def postJson[J: Writes](body: J = Json.obj()): Fox[Unit] = { request = request.addHttpHeaders(HeaderNames.CONTENT_TYPE -> jsonMimeType).withBody(Json.toJson(body)).withMethod("POST") - performRequest + performRequest.map(_ => ()) } def postProto[T <: GeneratedMessage](body: T): Fox[Unit] = { diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/slacknotification/SlackClient.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/slacknotification/SlackClient.scala index 99491288037..45f2d4e0766 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/slacknotification/SlackClient.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/slacknotification/SlackClient.scala @@ -45,6 +45,7 @@ class SlackClient(rpc: RPC, slackUri: String, name: String, verboseLoggingEnable rpc(slackUri).postJson( Json.obj("attachments" -> Json.arr(jsonMessage)) ) + () } else { logger.warn( s"Not sending slack notification as rate limit of $messagesSentSinceReset was reached. Message was: $jsonMessage") diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala index 70852b533a6..66976f7fd24 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala @@ -5,6 +5,7 @@ import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox +import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayer import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceId, DataSourceLike} import com.scalableminds.webknossos.datastore.rpc.RPC import com.scalableminds.webknossos.datastore.services.{ @@ -89,6 +90,12 @@ class TSRemoteWebknossosClient @Inject()( .getWithJsonResponse[String] ) ?~> "annotation.idForTracing.failed" + def updateAnnotationLayers(annotationId: String, annotationLayers: List[AnnotationLayer]): Fox[Unit] = + rpc(s"$webknossosUri/api/tracingstores/$tracingStoreName/updateAnnotationLayers") + .addQueryString("annotationId" -> annotationId) + .addQueryString("key" -> tracingStoreKey) + .postJson(annotationLayers) + override def requestUserAccess(accessRequest: UserAccessRequest)(implicit tc: TokenContext): Fox[UserAccessAnswer] = rpc(s"$webknossosUri/api/tracingstores/$tracingStoreName/validateUserAccess") .addQueryString("key" -> tracingStoreKey) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala index 33e81fcfe85..c69dfc1fb9d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala @@ -179,8 +179,21 @@ class AnnotationTransactionService @Inject()(handledGroupIdStore: TracingStoreRe } else failUnlessAlreadyHandled(updateGroup, annotationId, prevVersion) } } + _ <- applyImmediatelyIfNeeded(annotationId, updateGroups.flatMap(_.actions), newVersion) } yield newVersion + private def applyImmediatelyIfNeeded(annotationId: String, updates: List[UpdateAction], newVersion: Long)( + implicit ec: ExecutionContext, + tc: TokenContext): Fox[Unit] = + if (containsApplyImmediatelyUpdateActions(updates)) { + annotationService.get(annotationId, Some(newVersion)).map(_ => ()) + } else Fox.successful(()) + + private def containsApplyImmediatelyUpdateActions(updates: List[UpdateAction]) = updates.exists { + case _: ApplyImmediatelyUpdateAction => true + case _ => false + } + private def handleUpdateGroup(annotationId: String, updateActionGroup: UpdateActionGroup)( implicit ec: ExecutionContext, tc: TokenContext): Fox[Unit] = diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala index 860cb4f81b9..bda3fc5eead 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala @@ -18,7 +18,7 @@ object AnnotationLayerParameters { Json.using[WithDefaultValues].format[AnnotationLayerParameters] } -trait AnnotationUpdateAction extends UpdateAction +trait AnnotationUpdateAction extends ApplyImmediatelyUpdateAction case class AddLayerAnnotationUpdateAction(layerParameters: AnnotationLayerParameters, tracingId: String, @@ -48,7 +48,7 @@ case class DeleteLayerAnnotationUpdateAction(tracingId: String, } case class UpdateLayerMetadataAnnotationUpdateAction(tracingId: String, - layerName: String, // Just stored for nicer-looking history + layerName: String, actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index fff6b7890bf..e7abe01cb79 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -10,6 +10,7 @@ import com.scalableminds.webknossos.datastore.EditableMappingInfo.EditableMappin import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits +import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayer import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ EditableMappingLayer, EditableMappingService, @@ -428,6 +429,10 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss _ <- updatedWithNewVerson.flushBufferedUpdates() _ <- flushUpdatedTracings(updatedWithNewVerson) _ <- flushAnnotationInfo(annotationId, updatedWithNewVerson) + _ <- remoteWebknossosClient.updateAnnotationLayers(annotationId, + updatedWithNewVerson.annotation.layers + .map(AnnotationLayer.fromProto) + .toList) // TODO perf: skip if no layer changes } yield updatedWithNewVerson } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala index 6b56c292440..b13a71c58db 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala @@ -54,6 +54,8 @@ trait UpdateAction { def isViewOnlyChange: Boolean = false } +trait ApplyImmediatelyUpdateAction extends UpdateAction + trait LayerUpdateAction extends UpdateAction { def actionTracingId: String } From 6bd5ceec8a1940b4c4d7a9ccdbccdcafbcaff90e Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 21 Oct 2024 09:39:35 +0200 Subject: [PATCH 106/150] wip add layer --- app/controllers/AnnotationController.scala | 4 ++-- app/controllers/UserTokenController.scala | 4 ++-- .../WKRemoteTracingStoreController.scala | 4 ++-- .../TSRemoteWebknossosClient.scala | 7 +++++++ .../AnnotationTransactionService.scala | 7 ++++--- .../annotation/AnnotationUpdateActions.scala | 2 +- .../annotation/AnnotationWithTracings.scala | 9 ++++++--- .../annotation/TSAnnotationService.scala | 19 +++++++++++++++---- .../SkeletonTracingController.scala | 4 ++-- .../controllers/VolumeTracingController.scala | 7 ++++--- .../tracingstore/tracings/TracingId.scala | 11 +++++++++++ .../tracings/TracingService.scala | 9 +-------- 12 files changed, 57 insertions(+), 30 deletions(-) create mode 100644 webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingId.scala diff --git a/app/controllers/AnnotationController.scala b/app/controllers/AnnotationController.scala index e1b2dfa0128..87c2b734ca0 100755 --- a/app/controllers/AnnotationController.scala +++ b/app/controllers/AnnotationController.scala @@ -13,7 +13,7 @@ import com.scalableminds.webknossos.datastore.models.annotation.{ } import com.scalableminds.webknossos.datastore.rpc.RPC import com.scalableminds.webknossos.tracingstore.annotation.AnnotationLayerParameters -import com.scalableminds.webknossos.tracingstore.tracings.{TracingIds, TracingType} +import com.scalableminds.webknossos.tracingstore.tracings.{TracingId, TracingType} import mail.{MailchimpClient, MailchimpTag} import models.analytics.{AnalyticsService, CreateAnnotationEvent, OpenAnnotationEvent} import models.annotation.AnnotationState.Cancelled @@ -273,7 +273,7 @@ class AnnotationController @Inject()( ObjectId.dummyId, ObjectId.dummyId, List( - AnnotationLayer(TracingIds.dummyTracingId, + AnnotationLayer(TracingId.dummy, AnnotationLayerType.Skeleton, AnnotationLayer.defaultSkeletonLayerName, AnnotationLayerStatistics.unknown)) diff --git a/app/controllers/UserTokenController.scala b/app/controllers/UserTokenController.scala index 29580465eeb..22127329f93 100644 --- a/app/controllers/UserTokenController.scala +++ b/app/controllers/UserTokenController.scala @@ -11,7 +11,7 @@ import com.scalableminds.webknossos.datastore.services.{ UserAccessAnswer, UserAccessRequest } -import com.scalableminds.webknossos.tracingstore.tracings.TracingIds +import com.scalableminds.webknossos.tracingstore.tracings.TracingId import javax.inject.Inject import models.annotation._ @@ -161,7 +161,7 @@ class UserTokenController @Inject()(datasetDAO: DatasetDAO, mode: AccessMode, userBox: Box[User], token: Option[String]): Fox[UserAccessAnswer] = - if (tracingId == TracingIds.dummyTracingId) + if (tracingId == TracingId.dummy) Fox.successful(UserAccessAnswer(granted = true)) else for { diff --git a/app/controllers/WKRemoteTracingStoreController.scala b/app/controllers/WKRemoteTracingStoreController.scala index 14eb8e2146b..b64e4ed87d3 100644 --- a/app/controllers/WKRemoteTracingStoreController.scala +++ b/app/controllers/WKRemoteTracingStoreController.scala @@ -6,7 +6,7 @@ import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayer import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId import com.scalableminds.webknossos.tracingstore.TracingUpdatesReport -import com.scalableminds.webknossos.tracingstore.tracings.TracingIds +import com.scalableminds.webknossos.tracingstore.tracings.TracingId import javax.inject.Inject import models.analytics.{AnalyticsService, UpdateAnnotationEvent, UpdateAnnotationViewOnlyEvent} @@ -140,7 +140,7 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore Action.async { implicit request => tracingStoreService.validateAccess(name, key) { _ => implicit val ctx: DBAccessContext = GlobalAccessContext - if (tracingId == TracingIds.dummyTracingId) { + if (tracingId == TracingId.dummy) { Fox.successful(Ok(Json.toJson(ObjectId.dummyId))) } else { for { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala index 66976f7fd24..04dac6793a3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala @@ -5,6 +5,8 @@ import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox +import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing +import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayer import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceId, DataSourceLike} import com.scalableminds.webknossos.datastore.rpc.RPC @@ -14,6 +16,7 @@ import com.scalableminds.webknossos.datastore.services.{ UserAccessAnswer, UserAccessRequest } +import com.scalableminds.webknossos.tracingstore.annotation.AnnotationLayerParameters import com.typesafe.scalalogging.LazyLogging import play.api.inject.ApplicationLifecycle import play.api.libs.json.{JsObject, Json, OFormat} @@ -96,6 +99,10 @@ class TSRemoteWebknossosClient @Inject()( .addQueryString("key" -> tracingStoreKey) .postJson(annotationLayers) + def createTracingFor(annotationId: String, + layerParameters: AnnotationLayerParameters): Fox[Either[SkeletonTracing, VolumeTracing]] = + ??? // TODO + override def requestUserAccess(accessRequest: UserAccessRequest)(implicit tc: TokenContext): Fox[UserAccessAnswer] = rpc(s"$webknossosUri/api/tracingstores/$tracingStoreName/validateUserAccess") .addQueryString("key" -> tracingStoreKey) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala index c69dfc1fb9d..2399ed2549e 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala @@ -4,7 +4,7 @@ import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.tools.{Fox, JsonHelper} import com.scalableminds.util.tools.Fox.bool2Fox import com.scalableminds.webknossos.tracingstore.TracingStoreRedisStore -import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} +import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore, TracingId} import com.scalableminds.webknossos.tracingstore.tracings.volume.{ BucketMutatingVolumeUpdateAction, UpdateBucketVolumeAction, @@ -228,8 +228,9 @@ class AnnotationTransactionService @Inject()(handledGroupIdStore: TracingStoreRe case first :: rest => first.addInfo(updateActionGroup.info) :: rest } actionsWithInfo.map { - case a: UpdateBucketVolumeAction => a.withoutBase64Data - case a => a + case a: UpdateBucketVolumeAction => a.withoutBase64Data + case a: AddLayerAnnotationUpdateAction => a.copy(tracingId = Some(TracingId.generate)) + case a => a } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala index bda3fc5eead..f6ab1f62fee 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala @@ -21,7 +21,7 @@ object AnnotationLayerParameters { trait AnnotationUpdateAction extends ApplyImmediatelyUpdateAction case class AddLayerAnnotationUpdateAction(layerParameters: AnnotationLayerParameters, - tracingId: String, + tracingId: Option[String] = None, // filled in by backend eagerly on save actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index 658cac9da17..6996b398d6d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -78,14 +78,17 @@ case class AnnotationWithTracings( def version: Long = annotation.version - def addTracing(a: AddLayerAnnotationUpdateAction): AnnotationWithTracings = + def addLayer(a: AddLayerAnnotationUpdateAction, + tracingId: String, + tracing: Either[SkeletonTracing, VolumeTracing]): AnnotationWithTracings = this.copy( annotation = annotation.copy( layers = annotation.layers :+ AnnotationLayerProto( - a.tracingId, + tracingId, a.layerParameters.name.getOrElse(AnnotationLayer.defaultNameForType(a.layerParameters.typ)), `type` = AnnotationLayerType.toProto(a.layerParameters.typ) - )) + )), + tracingsById = tracingsById.updated(tracingId, tracing) ) def deleteTracing(a: DeleteLayerAnnotationUpdateAction): AnnotationWithTracings = diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index e7abe01cb79..a8911ed7e69 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -35,7 +35,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.{ FallbackDataHelper, KeyValueStoreImplicits, TracingDataStore, - TracingIds, + TracingId, TracingSelector, VersionedKeyValuePair } @@ -108,7 +108,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss for { updated <- updateAction match { case a: AddLayerAnnotationUpdateAction => - Fox.successful(annotationWithTracings.addTracing(a)) + addLayer(annotationId, annotationWithTracings, a) case a: DeleteLayerAnnotationUpdateAction => Fox.successful(annotationWithTracings.deleteTracing(a)) case a: UpdateLayerMetadataAnnotationUpdateAction => @@ -134,6 +134,17 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } } yield updated + private def addLayer( + annotationId: String, + annotationWithTracings: AnnotationWithTracings, + action: AddLayerAnnotationUpdateAction)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = + for { + _ <- Fox.successful(()) + tracingId <- action.tracingId ?~> "add layer action has no tracingId" + tracing <- remoteWebknossosClient.createTracingFor(annotationId, action.layerParameters) + updated = annotationWithTracings.addLayer(action, tracingId, tracing) + } yield updated + private def revertToVersion( annotationId: String, annotationWithTracings: AnnotationWithTracings, @@ -536,7 +547,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss version: Option[Long] = None, useCache: Boolean = true, applyUpdates: Boolean = false)(implicit tc: TokenContext, ec: ExecutionContext): Fox[VolumeTracing] = - if (tracingId == TracingIds.dummyTracingId) + if (tracingId == TracingId.dummy) Fox.successful(volumeTracingService.dummyTracing) else { for { @@ -551,7 +562,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss version: Option[Long] = None, useCache: Boolean = true, applyUpdates: Boolean = false)(implicit tc: TokenContext, ec: ExecutionContext): Fox[SkeletonTracing] = - if (tracingId == TracingIds.dummyTracingId) + if (tracingId == TracingId.dummy) Fox.successful(skeletonTracingService.dummyTracing) else { for { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala index fe1e105568e..f41279b4928 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala @@ -6,7 +6,7 @@ import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.SkeletonTracing.{SkeletonTracing, SkeletonTracingOpt, SkeletonTracings} import com.scalableminds.webknossos.datastore.services.UserAccessRequest import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService -import com.scalableminds.webknossos.tracingstore.tracings.TracingSelector +import com.scalableminds.webknossos.tracingstore.tracings.{TracingId, TracingSelector} import com.scalableminds.webknossos.tracingstore.tracings.skeleton._ import com.scalableminds.webknossos.tracingstore.tracings.volume.MergedVolumeStats import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingStoreAccessTokenService} @@ -106,7 +106,7 @@ class SkeletonTracingController @Inject()(skeletonTracingService: SkeletonTracin case (Some(tracing), Some(selector)) => Some((tracing, selector.tracingId)) case _ => None } - newTracingId = skeletonTracingService.generateTracingId + newTracingId = TracingId.generate mergedVolumeStats <- skeletonTracingService.mergeVolumeData(request.body.flatten, tracingsWithIds.map(_._1), newTracingId, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index f84c7ef7d53..75b8b61151b 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -35,7 +35,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ VolumeSegmentStatisticsService, VolumeTracingService } -import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingSelector} +import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingId, TracingSelector} import com.scalableminds.webknossos.tracingstore.{ TSRemoteDatastoreClient, TSRemoteWebknossosClient, @@ -146,7 +146,7 @@ class VolumeTracingController @Inject()( case (Some(tracing), Some(selector)) => Some((tracing, selector.tracingId)) case _ => None } - newTracingId = volumeTracingService.generateTracingId + newTracingId = TracingId.generate mergedVolumeStats <- volumeTracingService.mergeVolumeData(request.body.flatten, tracingsWithIds.map(_._1), newTracingId, @@ -299,10 +299,11 @@ class VolumeTracingController @Inject()( boundingBoxParsed <- Fox.runOptional(boundingBox)(BoundingBox.fromLiteral) remoteFallbackLayerOpt <- Fox.runIf(tracing.getHasEditableMapping)( volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId)) - newTracingId = volumeTracingService.generateTracingId + newTracingId = TracingId.generate // TODO /*_ <- Fox.runIf(tracing.getHasEditableMapping)( editableMappingService.duplicate(tracingId, newTracingId, version = None, remoteFallbackLayerOpt))*/ + // TODO actionTracingIds + addLayer tracing ids need to be remapped (as they need to be globally unique) (newId, newTracing) <- volumeTracingService.duplicate( annotationId, tracingId, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingId.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingId.scala new file mode 100644 index 00000000000..9c6a1af49eb --- /dev/null +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingId.scala @@ -0,0 +1,11 @@ +package com.scalableminds.webknossos.tracingstore.tracings + +import java.util.UUID + +object TracingId { + + def generate: String = UUID.randomUUID.toString + + lazy val dummy: String = "dummyTracingId" + +} diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala index afe6fd032a4..c0f06df09cc 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala @@ -10,14 +10,9 @@ import net.liftweb.common.Box import play.api.i18n.MessagesProvider import scalapb.{GeneratedMessage, GeneratedMessageCompanion} -import java.util.UUID import scala.concurrent.ExecutionContext import scala.concurrent.duration._ -object TracingIds { - val dummyTracingId: String = "dummyTracingId" -} - trait TracingService[T <: GeneratedMessage] extends KeyValueStoreImplicits with FoxImplicits @@ -65,10 +60,8 @@ trait TracingService[T <: GeneratedMessage] } */ - def generateTracingId: String = UUID.randomUUID.toString - def save(tracing: T, tracingId: Option[String], version: Long, toCache: Boolean = false): Fox[String] = { - val id = tracingId.getOrElse(generateTracingId) + val id = tracingId.getOrElse(TracingId.generate) if (toCache) { temporaryTracingStore.insert(id, tracing, Some(temporaryStoreTimeout)) temporaryTracingIdStore.insert(temporaryIdKey(id), "", Some(temporaryIdStoreTimeout)) From 40b7e435bf1d62dc0bb7c77eb83c976920407d1d Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 21 Oct 2024 10:59:51 +0200 Subject: [PATCH 107/150] create tracing proto on wk side --- app/controllers/AnnotationController.scala | 70 ----- .../WKRemoteTracingStoreController.scala | 22 ++ app/models/annotation/AnnotationService.scala | 255 ++++++++---------- conf/webknossos.latest.routes | 3 +- .../TSRemoteWebknossosClient.scala | 20 +- 5 files changed, 146 insertions(+), 224 deletions(-) diff --git a/app/controllers/AnnotationController.scala b/app/controllers/AnnotationController.scala index 87c2b734ca0..d5a1943f01f 100755 --- a/app/controllers/AnnotationController.scala +++ b/app/controllers/AnnotationController.scala @@ -183,54 +183,6 @@ class AnnotationController @Inject()( } yield JsonOk(json, Messages("annotation.isLockedByOwner.success")) } - def addAnnotationLayer(typ: String, id: String): Action[AnnotationLayerParameters] = - sil.SecuredAction.async(validateJson[AnnotationLayerParameters]) { implicit request => - for { - _ <- bool2Fox(AnnotationType.Explorational.toString == typ) ?~> "annotation.addLayer.explorationalsOnly" - restrictions <- provider.restrictionsFor(typ, id) ?~> "restrictions.notFound" ~> NOT_FOUND - _ <- restrictions.allowUpdate(request.identity) ?~> "notAllowed" ~> FORBIDDEN - annotation <- provider.provideAnnotation(typ, id, request.identity) - newLayerName = request.body.name.getOrElse(AnnotationLayer.defaultNameForType(request.body.typ)) - _ <- bool2Fox(!annotation.annotationLayers.exists(_.name == newLayerName)) ?~> "annotation.addLayer.nameInUse" - organization <- organizationDAO.findOne(request.identity._organization) - _ <- annotationService.addAnnotationLayer(annotation, organization._id, request.body) - updated <- provider.provideAnnotation(typ, id, request.identity) - json <- annotationService.publicWrites(updated, Some(request.identity)) ?~> "annotation.write.failed" - } yield JsonOk(json) - } - - def addAnnotationLayerWithoutType(id: String): Action[AnnotationLayerParameters] = - sil.SecuredAction.async(validateJson[AnnotationLayerParameters]) { implicit request => - for { - annotation <- provider.provideAnnotation(id, request.identity) ~> NOT_FOUND - result <- addAnnotationLayer(annotation.typ.toString, id)(request) - } yield result - } - - def deleteAnnotationLayer(typ: String, id: String, layerName: String): Action[AnyContent] = - sil.SecuredAction.async { implicit request => - for { - _ <- bool2Fox(AnnotationType.Explorational.toString == typ) ?~> "annotation.deleteLayer.explorationalsOnly" - annotation <- provider.provideAnnotation(typ, id, request.identity) - _ <- bool2Fox(annotation._user == request.identity._id) ?~> "notAllowed" ~> FORBIDDEN - layer <- annotation.annotationLayers.find(annotationLayer => annotationLayer.name == layerName) ?~> Messages( - "annotation.layer.notFound", - layerName) - _ <- bool2Fox(annotation.annotationLayers.length != 1) ?~> "annotation.deleteLayer.onlyLayer" - _ = logger.info( - s"Deleting annotation layer $layerName (tracing id ${layer.tracingId}, typ ${layer.typ}) for annotation $id") - _ <- annotationService.deleteAnnotationLayer(annotation, layerName) - } yield Ok - } - - def deleteAnnotationLayerWithoutType(id: String, layerName: String): Action[AnyContent] = - sil.SecuredAction.async { implicit request => - for { - annotation <- provider.provideAnnotation(id, request.identity) ~> NOT_FOUND - result <- deleteAnnotationLayer(annotation.typ.toString, id, layerName)(request) - } yield result - } - def createExplorational(organizationId: String, datasetName: String): Action[List[AnnotationLayerParameters]] = sil.SecuredAction.async(validateJson[List[AnnotationLayerParameters]]) { implicit request => for { @@ -282,28 +234,6 @@ class AnnotationController @Inject()( } yield JsonOk(json) } - def makeHybrid(typ: String, id: String, fallbackLayerName: Option[String]): Action[AnyContent] = - sil.SecuredAction.async { implicit request => - for { - _ <- bool2Fox(AnnotationType.Explorational.toString == typ) ?~> "annotation.addLayer.explorationalsOnly" - restrictions <- provider.restrictionsFor(typ, id) ?~> "restrictions.notFound" ~> NOT_FOUND - _ <- restrictions.allowUpdate(request.identity) ?~> "notAllowed" ~> FORBIDDEN - annotation <- provider.provideAnnotation(typ, id, request.identity) - organization <- organizationDAO.findOne(request.identity._organization) - _ <- annotationService.makeAnnotationHybrid(annotation, organization._id, fallbackLayerName) ?~> "annotation.makeHybrid.failed" - updated <- provider.provideAnnotation(typ, id, request.identity) - json <- annotationService.publicWrites(updated, Some(request.identity)) ?~> "annotation.write.failed" - } yield JsonOk(json) - } - - def makeHybridWithoutType(id: String, fallbackLayerName: Option[String]): Action[AnyContent] = - sil.SecuredAction.async { implicit request => - for { - annotation <- provider.provideAnnotation(id, request.identity) ~> NOT_FOUND - result <- makeHybrid(annotation.typ.toString, id, fallbackLayerName)(request) - } yield result - } - def downsample(typ: String, id: String, tracingId: String): Action[AnyContent] = sil.SecuredAction.async { implicit request => for { diff --git a/app/controllers/WKRemoteTracingStoreController.scala b/app/controllers/WKRemoteTracingStoreController.scala index b64e4ed87d3..2f4c966b8a1 100644 --- a/app/controllers/WKRemoteTracingStoreController.scala +++ b/app/controllers/WKRemoteTracingStoreController.scala @@ -3,9 +3,12 @@ package controllers import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext} import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.{Fox, FoxImplicits} +import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing +import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayer import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId import com.scalableminds.webknossos.tracingstore.TracingUpdatesReport +import com.scalableminds.webknossos.tracingstore.annotation.AnnotationLayerParameters import com.scalableminds.webknossos.tracingstore.tracings.TracingId import javax.inject.Inject @@ -16,6 +19,7 @@ import models.annotation.{ AnnotationDAO, AnnotationInformationProvider, AnnotationLayerDAO, + AnnotationService, TracingDataSourceTemporaryStore, TracingStoreService } @@ -26,6 +30,7 @@ import models.user.time.TimeSpanService import play.api.i18n.Messages import play.api.libs.json.Json import play.api.mvc.{Action, AnyContent, PlayBodyParsers} +import scalapb.GeneratedMessage import security.{WebknossosBearerTokenAuthenticatorService, WkSilhouetteEnvironment} import utils.{ObjectId, WkConf} @@ -39,6 +44,7 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore userDAO: UserDAO, annotationInformationProvider: AnnotationInformationProvider, analyticsService: AnalyticsService, + annotationService: AnnotationService, datasetDAO: DatasetDAO, annotationDAO: AnnotationDAO, annotationLayerDAO: AnnotationLayerDAO, @@ -168,4 +174,20 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore } yield Ok(Json.toJson(dataStore.url)) } } + + def createTracing(name: String, key: String, annotationId: String): Action[AnnotationLayerParameters] = + Action.async(validateJson[AnnotationLayerParameters]) { implicit request => + tracingStoreService.validateAccess(name, key) { _ => + implicit val ctx: DBAccessContext = GlobalAccessContext + for { + annotationIdValidated <- ObjectId.fromString(annotationId) + tracingEither <- annotationService.createTracingForExplorational(annotationIdValidated, request.body) + tracing: GeneratedMessage = tracingEither match { + case Left(s: SkeletonTracing) => s + case Right(v: VolumeTracing) => v + } + } yield Ok(tracing.toByteArray).as(protobufMimeType) + } + } + } diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index 57327d2924e..a6e34a8ec71 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -34,7 +34,6 @@ import com.scalableminds.webknossos.datastore.models.datasource.{ } import com.scalableminds.webknossos.datastore.rpc.RPC import com.scalableminds.webknossos.tracingstore.annotation.AnnotationLayerParameters -import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeDataZipFormat.VolumeDataZipFormat import com.scalableminds.webknossos.tracingstore.tracings.volume.{ ResolutionRestrictions, @@ -196,44 +195,34 @@ class AnnotationService @Inject()( VolumeTracingDefaults.largestSegmentId } - def addAnnotationLayer(annotation: Annotation, - organizationId: String, - annotationLayerParameters: AnnotationLayerParameters)(implicit ctx: DBAccessContext, - mp: MessagesProvider): Fox[Unit] = - for { - dataset <- datasetDAO.findOne(annotation._dataset) ?~> "dataset.notFoundForAnnotation" - dataSource <- datasetService.dataSourceFor(dataset).flatMap(_.toUsable) ?~> "dataSource.notFound" - newAnnotationLayers <- createTracingsForExplorational( - dataset, - dataSource, - annotation._id, - List(annotationLayerParameters), - organizationId, - annotation.annotationLayers) ?~> "annotation.createTracings.failed" - _ <- annotationLayersDAO.insertForAnnotation(annotation._id, newAnnotationLayers) - } yield () - - def deleteAnnotationLayer(annotation: Annotation, layerName: String): Fox[Unit] = - for { - _ <- annotationLayersDAO.deleteOneByName(annotation._id, layerName) - } yield () - - private def createTracingsForExplorational(dataset: Dataset, - dataSource: DataSource, - annotationId: ObjectId, - allAnnotationLayerParameters: List[AnnotationLayerParameters], - datasetOrganizationId: String, - existingAnnotationLayers: List[AnnotationLayer] = List())( + def createTracingForExplorational(annotationId: ObjectId, params: AnnotationLayerParameters)( implicit ctx: DBAccessContext, - mp: MessagesProvider): Fox[List[AnnotationLayer]] = { + mp: MessagesProvider): Fox[Either[SkeletonTracing, VolumeTracing]] = { + + def fetchOldPrecedenceLayer(existingAnnotationLayers: List[AnnotationLayer], + dataset: Dataset): Fox[Option[FetchedAnnotationLayer]] = + if (existingAnnotationLayers.isEmpty) Fox.successful(None) + else + for { + oldPrecedenceLayer <- selectLayerWithPrecedence(existingAnnotationLayers) + tracingStoreClient <- tracingStoreService.clientFor(dataset) + oldPrecedenceLayerFetched <- if (oldPrecedenceLayer.typ == AnnotationLayerType.Skeleton) + tracingStoreClient.getSkeletonTracing(oldPrecedenceLayer, None) + else + tracingStoreClient.getVolumeTracing(oldPrecedenceLayer, + None, + skipVolumeData = true, + volumeDataZipFormat = VolumeDataZipFormat.wkw, + dataset.voxelSize) + } yield Some(oldPrecedenceLayerFetched) - def getAutoFallbackLayerName: Option[String] = + def getAutoFallbackLayerName(dataSource: DataSource): Option[String] = dataSource.dataLayers.find { case _: SegmentationLayer => true case _ => false }.map(_.name) - def getFallbackLayer(fallbackLayerName: String): Fox[SegmentationLayer] = + def getFallbackLayer(dataSource: DataSource, fallbackLayerName: String): Fox[SegmentationLayer] = for { fallbackLayer <- dataSource.dataLayers .filter(dl => dl.name == fallbackLayerName) @@ -251,87 +240,6 @@ class AnnotationService @Inject()( fallbackLayer.elementClass) } yield fallbackLayer - def createAndSaveAnnotationLayer(annotationLayerParameters: AnnotationLayerParameters, - oldPrecedenceLayerProperties: Option[RedundantTracingProperties], - dataStore: DataStore): Fox[AnnotationLayer] = - for { - client <- tracingStoreService.clientFor(dataset) - tracingIdAndName <- annotationLayerParameters.typ match { - case AnnotationLayerType.Skeleton => - val skeleton = SkeletonTracingDefaults.createInstance.copy( - datasetName = dataset.name, - editPosition = dataSource.center, - organizationId = Some(datasetOrganizationId), - additionalAxes = AdditionalAxis.toProto(dataSource.additionalAxesUnion) - ) - val skeletonAdapted = oldPrecedenceLayerProperties.map { p => - skeleton.copy( - editPosition = p.editPosition, - editRotation = p.editRotation, - zoomLevel = p.zoomLevel, - userBoundingBoxes = p.userBoundingBoxes, - editPositionAdditionalCoordinates = p.editPositionAdditionalCoordinates - ) - }.getOrElse(skeleton) - for { - tracingId <- client.saveSkeletonTracing(skeletonAdapted) - name = annotationLayerParameters.name.getOrElse( - AnnotationLayer.defaultNameForType(annotationLayerParameters.typ)) - } yield (tracingId, name) - case AnnotationLayerType.Volume => - val autoFallbackLayerName = - if (annotationLayerParameters.autoFallbackLayer) getAutoFallbackLayerName else None - val fallbackLayerName = annotationLayerParameters.fallbackLayerName.orElse(autoFallbackLayerName) - for { - fallbackLayer <- Fox.runOptional(fallbackLayerName)(getFallbackLayer) - volumeTracing <- createVolumeTracing( - dataSource, - datasetOrganizationId, - dataStore, - fallbackLayer, - resolutionRestrictions = - annotationLayerParameters.resolutionRestrictions.getOrElse(ResolutionRestrictions.empty), - mappingName = annotationLayerParameters.mappingName - ) - volumeTracingAdapted = oldPrecedenceLayerProperties.map { p => - volumeTracing.copy( - editPosition = p.editPosition, - editRotation = p.editRotation, - zoomLevel = p.zoomLevel, - userBoundingBoxes = p.userBoundingBoxes, - editPositionAdditionalCoordinates = p.editPositionAdditionalCoordinates - ) - }.getOrElse(volumeTracing) - volumeTracingId <- client.saveVolumeTracing(volumeTracingAdapted, dataSource = Some(dataSource)) - name = annotationLayerParameters.name - .orElse(autoFallbackLayerName) - .getOrElse(AnnotationLayer.defaultNameForType(annotationLayerParameters.typ)) - } yield (volumeTracingId, name) - case _ => - Fox.failure(s"Unknown AnnotationLayerType: ${annotationLayerParameters.typ}") - } - } yield - AnnotationLayer(tracingIdAndName._1, - annotationLayerParameters.typ, - tracingIdAndName._2, - AnnotationLayerStatistics.zeroedForType(annotationLayerParameters.typ)) - - def fetchOldPrecedenceLayer: Fox[Option[FetchedAnnotationLayer]] = - if (existingAnnotationLayers.isEmpty) Fox.successful(None) - else - for { - oldPrecedenceLayer <- selectLayerWithPrecedence(existingAnnotationLayers) - tracingStoreClient <- tracingStoreService.clientFor(dataset) - oldPrecedenceLayerFetched <- if (oldPrecedenceLayer.typ == AnnotationLayerType.Skeleton) - tracingStoreClient.getSkeletonTracing(oldPrecedenceLayer, None) - else - tracingStoreClient.getVolumeTracing(oldPrecedenceLayer, - None, - skipVolumeData = true, - volumeDataZipFormat = VolumeDataZipFormat.wkw, - dataset.voxelSize) - } yield Some(oldPrecedenceLayerFetched) - def extractPrecedenceProperties(oldPrecedenceLayer: FetchedAnnotationLayer): RedundantTracingProperties = oldPrecedenceLayer.tracing match { case Left(s) => @@ -354,6 +262,84 @@ class AnnotationService @Inject()( ) } + for { + annotation <- annotationDAO.findOne(annotationId) + dataset <- datasetDAO.findOne(annotation._dataset) + dataStore <- dataStoreDAO.findOneByName(dataset._dataStore.trim) ?~> "dataStore.notFoundForDataset" + inboxDataSource <- datasetService.dataSourceFor(dataset) + dataSource <- inboxDataSource.toUsable ?~> Messages("dataset.notImported", inboxDataSource.id.name) + oldPrecedenceLayer <- fetchOldPrecedenceLayer(annotation.annotationLayers, dataset) + oldPrecedenceLayerProperties: Option[RedundantTracingProperties] = oldPrecedenceLayer.map( + extractPrecedenceProperties) + tracing <- params.typ match { + case AnnotationLayerType.Skeleton => + val skeleton = SkeletonTracingDefaults.createInstance.copy( + datasetName = dataset.name, + editPosition = dataSource.center, + organizationId = Some(dataset._organization), + additionalAxes = AdditionalAxis.toProto(dataSource.additionalAxesUnion) + ) + val skeletonAdapted = oldPrecedenceLayerProperties.map { p: RedundantTracingProperties => + skeleton.copy( + editPosition = p.editPosition, + editRotation = p.editRotation, + zoomLevel = p.zoomLevel, + userBoundingBoxes = p.userBoundingBoxes, + editPositionAdditionalCoordinates = p.editPositionAdditionalCoordinates + ) + }.getOrElse(skeleton) + Fox.successful(Left(skeletonAdapted)) + case AnnotationLayerType.Volume => + val autoFallbackLayerName = + if (params.autoFallbackLayer) getAutoFallbackLayerName(dataSource) else None + val fallbackLayerName = params.fallbackLayerName.orElse(autoFallbackLayerName) + for { + fallbackLayer <- Fox.runOptional(fallbackLayerName)(n => getFallbackLayer(dataSource, n)) + volumeTracing <- createVolumeTracing( + dataSource, + dataset._organization, + dataStore, + fallbackLayer, + resolutionRestrictions = params.resolutionRestrictions.getOrElse(ResolutionRestrictions.empty), + mappingName = params.mappingName + ) + volumeTracingAdapted = oldPrecedenceLayerProperties.map { p: RedundantTracingProperties => + volumeTracing.copy( + editPosition = p.editPosition, + editRotation = p.editRotation, + zoomLevel = p.zoomLevel, + userBoundingBoxes = p.userBoundingBoxes, + editPositionAdditionalCoordinates = p.editPositionAdditionalCoordinates + ) + }.getOrElse(volumeTracing) + } yield Right(volumeTracingAdapted) + } + } yield tracing + } + + private def createLayersForExplorational(dataset: Dataset, + annotationId: ObjectId, + allAnnotationLayerParameters: List[AnnotationLayerParameters], + existingAnnotationLayers: List[AnnotationLayer])( + implicit ctx: DBAccessContext, + mp: MessagesProvider): Fox[List[AnnotationLayer]] = { + + def createAndSaveAnnotationLayer(annotationLayerParameters: AnnotationLayerParameters): Fox[AnnotationLayer] = + for { + client <- tracingStoreService.clientFor(dataset) + tracing <- createTracingForExplorational(annotationId, annotationLayerParameters) + layerName = annotationLayerParameters.name.getOrElse( + AnnotationLayer.defaultNameForType(annotationLayerParameters.typ)) + tracingId <- tracing match { + case Left(skeleton) => client.saveSkeletonTracing(skeleton) + case Right(volume) => client.saveVolumeTracing(volume) + } + } yield + AnnotationLayer(tracingId, + annotationLayerParameters.typ, + layerName, + AnnotationLayerStatistics.zeroedForType(annotationLayerParameters.typ)) + def createAndSaveAnnotationProto(annotationId: ObjectId, annotationLayers: List[AnnotationLayer]): Fox[Unit] = { val layersProto = annotationLayers.map { l => AnnotationLayerProto( @@ -362,7 +348,7 @@ class AnnotationService @Inject()( AnnotationLayerType.toProto(l.typ) ) } - // todo pass right name, description here + //TODO pass right name, description here val annotationProto = AnnotationProto(name = None, description = None, version = 0L, layers = layersProto) for { tracingStoreClient <- tracingStoreService.clientFor(dataset) @@ -380,11 +366,7 @@ class AnnotationService @Inject()( We do this for *every* new layer, since we only later get its ID which determines the actual precedence. All of this is skipped if existingAnnotationLayers is empty. */ - oldPrecedenceLayer <- fetchOldPrecedenceLayer - dataStore <- dataStoreDAO.findOneByName(dataset._dataStore.trim) ?~> "dataStore.notFoundForDataset" - precedenceProperties = oldPrecedenceLayer.map(extractPrecedenceProperties) - newAnnotationLayers <- Fox.serialCombined(allAnnotationLayerParameters)(p => - createAndSaveAnnotationLayer(p, precedenceProperties, dataStore)) + newAnnotationLayers <- Fox.serialCombined(allAnnotationLayerParameters)(createAndSaveAnnotationLayer) _ <- createAndSaveAnnotationProto(annotationId, newAnnotationLayers) } yield newAnnotationLayers } @@ -410,42 +392,17 @@ class AnnotationService @Inject()( m: MessagesProvider): Fox[Annotation] = for { dataset <- datasetDAO.findOne(datasetId) ?~> "dataset.noAccessById" - dataSource <- datasetService.dataSourceFor(dataset) - datasetOrganization <- organizationDAO.findOne(dataset._organization)(GlobalAccessContext) ?~> "organization.notFound" - usableDataSource <- dataSource.toUsable ?~> Messages("dataset.notImported", dataSource.id.name) newAnnotationId = ObjectId.generate - annotationLayers <- createTracingsForExplorational(dataset, - usableDataSource, - newAnnotationId, - annotationLayerParameters, - datasetOrganization._id) ?~> "annotation.createTracings.failed" + annotationLayers <- createLayersForExplorational( + dataset, + newAnnotationId, + annotationLayerParameters, + existingAnnotationLayers = List.empty) ?~> "annotation.createTracings.failed" teamId <- selectSuitableTeam(user, dataset) ?~> "annotation.create.forbidden" annotation = Annotation(newAnnotationId, datasetId, None, teamId, user._id, annotationLayers) _ <- annotationDAO.insertOne(annotation) } yield annotation - def makeAnnotationHybrid(annotation: Annotation, organizationId: String, fallbackLayerName: Option[String])( - implicit ctx: DBAccessContext, - mp: MessagesProvider): Fox[Unit] = - for { - newAnnotationLayerType <- annotation.tracingType match { - case TracingType.skeleton => Fox.successful(AnnotationLayerType.Volume) - case TracingType.volume => Fox.successful(AnnotationLayerType.Skeleton) - case _ => Fox.failure("annotation.makeHybrid.alreadyHybrid") - } - usedFallbackLayerName = if (newAnnotationLayerType == AnnotationLayerType.Volume) fallbackLayerName else None - newAnnotationLayerParameters = AnnotationLayerParameters( - newAnnotationLayerType, - usedFallbackLayerName, - autoFallbackLayer = false, - None, - Some(ResolutionRestrictions.empty), - Some(AnnotationLayer.defaultNameForType(newAnnotationLayerType)), - None - ) - _ <- addAnnotationLayer(annotation, organizationId, newAnnotationLayerParameters) ?~> "makeHybrid.createTracings.failed" - } yield () - def downsampleAnnotation(annotation: Annotation, volumeAnnotationLayer: AnnotationLayer)( implicit ctx: DBAccessContext): Fox[Unit] = for { diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes index 64fdcd470a4..88bd96335ac 100644 --- a/conf/webknossos.latest.routes +++ b/conf/webknossos.latest.routes @@ -128,6 +128,7 @@ GET /tracingstores/:name/dataSource GET /tracingstores/:name/dataSourceId controllers.WKRemoteTracingStoreController.dataSourceIdForTracing(name: String, key: String, tracingId: String) GET /tracingstores/:name/annotationId controllers.WKRemoteTracingStoreController.annotationIdForTracing(name: String, key: String, tracingId: String) GET /tracingstores/:name/dataStoreUri/:datasetName controllers.WKRemoteTracingStoreController.dataStoreUriForDataset(name: String, key: String, organizationId: Option[String], datasetName: String) +POST /tracingstores/:name/createTracing controllers.WKRemoteTracingStoreController.createTracing(name: String, key: String, annotationId: String) # User access tokens for datastore authentication POST /userToken/generate controllers.UserTokenController.generateTokenForDataStore() @@ -146,7 +147,6 @@ PATCH /annotations/:typ/:id/transfer PATCH /annotations/:typ/:id/editLockedState controllers.AnnotationController.editLockedState(typ: String, id: String, isLockedByOwner: Boolean) GET /annotations/:id/info controllers.AnnotationController.infoWithoutType(id: String, timestamp: Option[Long]) -PATCH /annotations/:id/makeHybrid controllers.AnnotationController.makeHybridWithoutType(id: String, fallbackLayerName: Option[String]) PATCH /annotations/:id/downsample controllers.AnnotationController.downsampleWithoutType(id: String, tracingId: String) PATCH /annotations/:id/addAnnotationLayer controllers.AnnotationController.addAnnotationLayerWithoutType(id: String) PATCH /annotations/:id/deleteAnnotationLayer controllers.AnnotationController.deleteAnnotationLayerWithoutType(id: String, layerName: String) @@ -157,7 +157,6 @@ POST /annotations/:id/acquireMutex PATCH /annotations/addSegmentIndicesToAll controllers.AnnotationController.addSegmentIndicesToAll(parallelBatchCount: Int, dryRun: Boolean, skipTracings: Option[String]) GET /annotations/:typ/:id/info controllers.AnnotationController.info(typ: String, id: String, timestamp: Option[Long]) -PATCH /annotations/:typ/:id/makeHybrid controllers.AnnotationController.makeHybrid(typ: String, id: String, fallbackLayerName: Option[String]) PATCH /annotations/:typ/:id/downsample controllers.AnnotationController.downsample(typ: String, id: String, tracingId: String) PATCH /annotations/:typ/:id/addAnnotationLayer controllers.AnnotationController.addAnnotationLayer(typ: String, id: String) PATCH /annotations/:typ/:id/deleteAnnotationLayer controllers.AnnotationController.deleteAnnotationLayer(typ: String, id: String, layerName: String) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala index 04dac6793a3..43f73e1a87f 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala @@ -7,7 +7,7 @@ import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing -import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayer +import com.scalableminds.webknossos.datastore.models.annotation.{AnnotationLayer, AnnotationLayerType} import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceId, DataSourceLike} import com.scalableminds.webknossos.datastore.rpc.RPC import com.scalableminds.webknossos.datastore.services.{ @@ -17,6 +17,7 @@ import com.scalableminds.webknossos.datastore.services.{ UserAccessRequest } import com.scalableminds.webknossos.tracingstore.annotation.AnnotationLayerParameters +import com.scalableminds.webknossos.tracingstore.tracings.TracingType import com.typesafe.scalalogging.LazyLogging import play.api.inject.ApplicationLifecycle import play.api.libs.json.{JsObject, Json, OFormat} @@ -100,8 +101,21 @@ class TSRemoteWebknossosClient @Inject()( .postJson(annotationLayers) def createTracingFor(annotationId: String, - layerParameters: AnnotationLayerParameters): Fox[Either[SkeletonTracing, VolumeTracing]] = - ??? // TODO + layerParameters: AnnotationLayerParameters): Fox[Either[SkeletonTracing, VolumeTracing]] = { + val req = rpc(s"$webknossosUri/api/tracingstores/$tracingStoreName/createTracing") + .addQueryString("annotationId" -> annotationId) + .addQueryString("key" -> tracingStoreKey) + layerParameters.typ match { + case AnnotationLayerType.Volume => + req + .postJsonWithProtoResponse[AnnotationLayerParameters, VolumeTracing](layerParameters)(VolumeTracing) + .map(Right(_)) + case AnnotationLayerType.Skeleton => + req + .postJsonWithProtoResponse[AnnotationLayerParameters, SkeletonTracing](layerParameters)(SkeletonTracing) + .map(Left(_)) + } + } override def requestUserAccess(accessRequest: UserAccessRequest)(implicit tc: TokenContext): Fox[UserAccessAnswer] = rpc(s"$webknossosUri/api/tracingstores/$tracingStoreName/validateUserAccess") From 6b3dd0faf882a5531d0b8c7f2341b4d980594e1a Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 21 Oct 2024 11:24:09 +0200 Subject: [PATCH 108/150] fix creating annotation layers for fresh annotation --- app/controllers/AnnotationIOController.scala | 6 +- .../WKRemoteTracingStoreController.scala | 6 +- .../AnnotationLayerPrecedence.scala | 143 +++++++++++++ app/models/annotation/AnnotationService.scala | 191 ++++-------------- app/models/annotation/nml/NmlWriter.scala | 15 +- conf/webknossos.latest.routes | 4 - 6 files changed, 196 insertions(+), 169 deletions(-) create mode 100644 app/models/annotation/AnnotationLayerPrecedence.scala diff --git a/app/controllers/AnnotationIOController.scala b/app/controllers/AnnotationIOController.scala index a3e2bdf490c..c06dc1bb7a0 100755 --- a/app/controllers/AnnotationIOController.scala +++ b/app/controllers/AnnotationIOController.scala @@ -79,6 +79,7 @@ class AnnotationIOController @Inject()( extends Controller with FoxImplicits with ProtoGeometryImplicits + with AnnotationLayerPrecedence with LazyLogging { implicit val actorSystem: ActorSystem = ActorSystem() @@ -332,9 +333,8 @@ class AnnotationIOController @Inject()( boundingBox = bbox, elementClass = elementClass, fallbackLayer = fallbackLayerOpt.map(_.name), - largestSegmentId = - annotationService.combineLargestSegmentIdsByPrecedence(volumeTracing.largestSegmentId, - fallbackLayerOpt.map(_.largestSegmentId)), + largestSegmentId = combineLargestSegmentIdsByPrecedence(volumeTracing.largestSegmentId, + fallbackLayerOpt.map(_.largestSegmentId)), resolutions = VolumeTracingDownsampling.magsForVolumeTracing(dataSource, fallbackLayerOpt).map(vec3IntToProto), hasSegmentIndex = Some(tracingCanHaveSegmentIndex) ) diff --git a/app/controllers/WKRemoteTracingStoreController.scala b/app/controllers/WKRemoteTracingStoreController.scala index 2f4c966b8a1..4ce52bdef84 100644 --- a/app/controllers/WKRemoteTracingStoreController.scala +++ b/app/controllers/WKRemoteTracingStoreController.scala @@ -181,7 +181,11 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore implicit val ctx: DBAccessContext = GlobalAccessContext for { annotationIdValidated <- ObjectId.fromString(annotationId) - tracingEither <- annotationService.createTracingForExplorational(annotationIdValidated, request.body) + annotation <- annotationDAO.findOne(annotationIdValidated) ?~> "annotation.notFound" + dataset <- datasetDAO.findOne(annotation._dataset) + tracingEither <- annotationService.createTracingForExplorational(dataset, + request.body, + annotation.annotationLayers) tracing: GeneratedMessage = tracingEither match { case Left(s: SkeletonTracing) => s case Right(v: VolumeTracing) => v diff --git a/app/models/annotation/AnnotationLayerPrecedence.scala b/app/models/annotation/AnnotationLayerPrecedence.scala new file mode 100644 index 00000000000..8d255cae38c --- /dev/null +++ b/app/models/annotation/AnnotationLayerPrecedence.scala @@ -0,0 +1,143 @@ +package models.annotation + +import com.scalableminds.util.tools.Fox +import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing +import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing +import com.scalableminds.webknossos.datastore.geometry.{ + AdditionalCoordinateProto, + NamedBoundingBoxProto, + Vec3DoubleProto, + Vec3IntProto +} +import com.scalableminds.webknossos.datastore.models.annotation.{ + AnnotationLayer, + AnnotationLayerType, + FetchedAnnotationLayer +} +import com.scalableminds.webknossos.tracingstore.tracings.volume.{VolumeDataZipFormat, VolumeTracingDefaults} +import models.dataset.Dataset + +import scala.concurrent.ExecutionContext + +// Used to pass duplicate properties when creating a new tracing to avoid masking them. +// Uses the proto-generated geometry classes, hence the full qualifiers. +case class RedundantTracingProperties( + editPosition: Vec3IntProto, + editRotation: Vec3DoubleProto, + zoomLevel: Double, + userBoundingBoxes: Seq[NamedBoundingBoxProto], + editPositionAdditionalCoordinates: Seq[AdditionalCoordinateProto], +) + +trait AnnotationLayerPrecedence { + + protected def combineLargestSegmentIdsByPrecedence(fromNml: Option[Long], + fromFallbackLayer: Option[Option[Long]]): Option[Long] = + if (fromNml.nonEmpty) + // This was called for an NML upload. The NML had an explicit largestSegmentId. Use that. + fromNml + else if (fromFallbackLayer.nonEmpty) + // There is a fallback layer. Use its largestSegmentId, even if it is None. + // Some tracing functionality will be disabled until a segment id is set by the user. + fromFallbackLayer.flatten + else { + // There is no fallback layer. Start at default segment id for fresh volume layers + VolumeTracingDefaults.largestSegmentId + } + + protected def adaptSkeletonTracing( + skeletonTracing: SkeletonTracing, + oldPrecedenceLayerProperties: Option[RedundantTracingProperties]): SkeletonTracing = + oldPrecedenceLayerProperties.map { p: RedundantTracingProperties => + skeletonTracing.copy( + editPosition = p.editPosition, + editRotation = p.editRotation, + zoomLevel = p.zoomLevel, + userBoundingBoxes = p.userBoundingBoxes, + editPositionAdditionalCoordinates = p.editPositionAdditionalCoordinates + ) + }.getOrElse(skeletonTracing) + + protected def adaptVolumeTracing(volumeTracing: VolumeTracing, + oldPrecedenceLayerProperties: Option[RedundantTracingProperties]): VolumeTracing = + oldPrecedenceLayerProperties.map { p: RedundantTracingProperties => + volumeTracing.copy( + editPosition = p.editPosition, + editRotation = p.editRotation, + zoomLevel = p.zoomLevel, + userBoundingBoxes = p.userBoundingBoxes, + editPositionAdditionalCoordinates = p.editPositionAdditionalCoordinates + ) + }.getOrElse(volumeTracing) + + protected def getOldPrecedenceLayerProperties(existingAnnotationLayers: List[AnnotationLayer], + dataset: Dataset, + tracingStoreClient: WKRemoteTracingStoreClient)( + implicit ec: ExecutionContext): Fox[Option[RedundantTracingProperties]] = + for { + oldPrecedenceLayer <- fetchOldPrecedenceLayer(existingAnnotationLayers, dataset, tracingStoreClient) + oldPrecedenceLayerProperties: Option[RedundantTracingProperties] = oldPrecedenceLayer.map( + extractPrecedenceProperties) + } yield oldPrecedenceLayerProperties + + // If there is more than one tracing, select the one that has precedence for the parameters (they should be identical anyway) + protected def selectLayerWithPrecedenceFetched( + skeletonLayers: List[FetchedAnnotationLayer], + volumeLayers: List[FetchedAnnotationLayer])(implicit ec: ExecutionContext): Fox[FetchedAnnotationLayer] = + if (skeletonLayers.nonEmpty) { + Fox.successful(skeletonLayers.minBy(_.tracingId)) + } else if (volumeLayers.nonEmpty) { + Fox.successful(volumeLayers.minBy(_.tracingId)) + } else Fox.failure("annotation.download.noLayers") + + private def selectLayerWithPrecedence(annotationLayers: List[AnnotationLayer])( + implicit ec: ExecutionContext): Fox[AnnotationLayer] = { + val skeletonLayers = annotationLayers.filter(_.typ == AnnotationLayerType.Skeleton) + val volumeLayers = annotationLayers.filter(_.typ == AnnotationLayerType.Volume) + if (skeletonLayers.nonEmpty) { + Fox.successful(skeletonLayers.minBy(_.tracingId)) + } else if (volumeLayers.nonEmpty) { + Fox.successful(volumeLayers.minBy(_.tracingId)) + } else Fox.failure("Trying to select precedence layer from empty layer list.") + } + + private def fetchOldPrecedenceLayer(existingAnnotationLayers: List[AnnotationLayer], + dataset: Dataset, + tracingStoreClient: WKRemoteTracingStoreClient)( + implicit ec: ExecutionContext): Fox[Option[FetchedAnnotationLayer]] = + if (existingAnnotationLayers.isEmpty) Fox.successful(None) + else + for { + oldPrecedenceLayer <- selectLayerWithPrecedence(existingAnnotationLayers) + oldPrecedenceLayerFetched <- if (oldPrecedenceLayer.typ == AnnotationLayerType.Skeleton) + tracingStoreClient.getSkeletonTracing(oldPrecedenceLayer, None) + else + tracingStoreClient.getVolumeTracing(oldPrecedenceLayer, + None, + skipVolumeData = true, + volumeDataZipFormat = VolumeDataZipFormat.wkw, + dataset.voxelSize) + } yield Some(oldPrecedenceLayerFetched) + + private def extractPrecedenceProperties(oldPrecedenceLayer: FetchedAnnotationLayer): RedundantTracingProperties = + oldPrecedenceLayer.tracing match { + case Left(s) => + RedundantTracingProperties( + s.editPosition, + s.editRotation, + s.zoomLevel, + s.userBoundingBoxes ++ s.userBoundingBox.map( + com.scalableminds.webknossos.datastore.geometry.NamedBoundingBoxProto(0, None, None, None, _)), + s.editPositionAdditionalCoordinates + ) + case Right(v) => + RedundantTracingProperties( + v.editPosition, + v.editRotation, + v.zoomLevel, + v.userBoundingBoxes ++ v.userBoundingBox.map( + com.scalableminds.webknossos.datastore.geometry.NamedBoundingBoxProto(0, None, None, None, _)), + v.editPositionAdditionalCoordinates + ) + } +} diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index a6e34a8ec71..7c22ca76ec0 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -10,13 +10,7 @@ import com.scalableminds.util.tools.{BoxImplicits, Fox, FoxImplicits, TextUtils} import com.scalableminds.webknossos.datastore.Annotation.{AnnotationLayerProto, AnnotationProto} import com.scalableminds.webknossos.datastore.SkeletonTracing._ import com.scalableminds.webknossos.datastore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings} -import com.scalableminds.webknossos.datastore.geometry.{ - AdditionalCoordinateProto, - ColorProto, - NamedBoundingBoxProto, - Vec3DoubleProto, - Vec3IntProto -} +import com.scalableminds.webknossos.datastore.geometry.ColorProto import com.scalableminds.webknossos.datastore.helpers.{NodeDefaults, ProtoGeometryImplicits, SkeletonTracingDefaults} import com.scalableminds.webknossos.datastore.models.VoxelSize import com.scalableminds.webknossos.datastore.models.annotation.{ @@ -37,7 +31,6 @@ import com.scalableminds.webknossos.tracingstore.annotation.AnnotationLayerParam import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeDataZipFormat.VolumeDataZipFormat import com.scalableminds.webknossos.tracingstore.tracings.volume.{ ResolutionRestrictions, - VolumeDataZipFormat, VolumeTracingDefaults, VolumeTracingDownsampling } @@ -75,16 +68,6 @@ case class DownloadAnnotation(skeletonTracingIdOpt: Option[String], organizationId: String, datasetName: String) -// Used to pass duplicate properties when creating a new tracing to avoid masking them. -// Uses the proto-generated geometry classes, hence the full qualifiers. -case class RedundantTracingProperties( - editPosition: Vec3IntProto, - editRotation: Vec3DoubleProto, - zoomLevel: Double, - userBoundingBoxes: Seq[NamedBoundingBoxProto], - editPositionAdditionalCoordinates: Seq[AdditionalCoordinateProto], -) - class AnnotationService @Inject()( annotationInformationProvider: AnnotationInformationProvider, savedTracingInformationHandler: SavedTracingInformationHandler, @@ -114,6 +97,7 @@ class AnnotationService @Inject()( extends BoxImplicits with FoxImplicits with ProtoGeometryImplicits + with AnnotationLayerPrecedence with LazyLogging { implicit val actorSystem: ActorSystem = ActorSystem() @@ -181,41 +165,12 @@ class AnnotationService @Inject()( ) } - def combineLargestSegmentIdsByPrecedence(fromNml: Option[Long], - fromFallbackLayer: Option[Option[Long]]): Option[Long] = - if (fromNml.nonEmpty) - // This was called for an NML upload. The NML had an explicit largestSegmentId. Use that. - fromNml - else if (fromFallbackLayer.nonEmpty) - // There is a fallback layer. Use its largestSegmentId, even if it is None. - // Some tracing functionality will be disabled until a segment id is set by the user. - fromFallbackLayer.flatten - else { - // There is no fallback layer. Start at default segment id for fresh volume layers - VolumeTracingDefaults.largestSegmentId - } - - def createTracingForExplorational(annotationId: ObjectId, params: AnnotationLayerParameters)( + def createTracingForExplorational(dataset: Dataset, + params: AnnotationLayerParameters, + existingAnnotationLayers: List[AnnotationLayer])( implicit ctx: DBAccessContext, mp: MessagesProvider): Fox[Either[SkeletonTracing, VolumeTracing]] = { - def fetchOldPrecedenceLayer(existingAnnotationLayers: List[AnnotationLayer], - dataset: Dataset): Fox[Option[FetchedAnnotationLayer]] = - if (existingAnnotationLayers.isEmpty) Fox.successful(None) - else - for { - oldPrecedenceLayer <- selectLayerWithPrecedence(existingAnnotationLayers) - tracingStoreClient <- tracingStoreService.clientFor(dataset) - oldPrecedenceLayerFetched <- if (oldPrecedenceLayer.typ == AnnotationLayerType.Skeleton) - tracingStoreClient.getSkeletonTracing(oldPrecedenceLayer, None) - else - tracingStoreClient.getVolumeTracing(oldPrecedenceLayer, - None, - skipVolumeData = true, - volumeDataZipFormat = VolumeDataZipFormat.wkw, - dataset.voxelSize) - } yield Some(oldPrecedenceLayerFetched) - def getAutoFallbackLayerName(dataSource: DataSource): Option[String] = dataSource.dataLayers.find { case _: SegmentationLayer => true @@ -240,37 +195,24 @@ class AnnotationService @Inject()( fallbackLayer.elementClass) } yield fallbackLayer - def extractPrecedenceProperties(oldPrecedenceLayer: FetchedAnnotationLayer): RedundantTracingProperties = - oldPrecedenceLayer.tracing match { - case Left(s) => - RedundantTracingProperties( - s.editPosition, - s.editRotation, - s.zoomLevel, - s.userBoundingBoxes ++ s.userBoundingBox.map( - com.scalableminds.webknossos.datastore.geometry.NamedBoundingBoxProto(0, None, None, None, _)), - s.editPositionAdditionalCoordinates - ) - case Right(v) => - RedundantTracingProperties( - v.editPosition, - v.editRotation, - v.zoomLevel, - v.userBoundingBoxes ++ v.userBoundingBox.map( - com.scalableminds.webknossos.datastore.geometry.NamedBoundingBoxProto(0, None, None, None, _)), - v.editPositionAdditionalCoordinates - ) - } - for { - annotation <- annotationDAO.findOne(annotationId) - dataset <- datasetDAO.findOne(annotation._dataset) dataStore <- dataStoreDAO.findOneByName(dataset._dataStore.trim) ?~> "dataStore.notFoundForDataset" inboxDataSource <- datasetService.dataSourceFor(dataset) dataSource <- inboxDataSource.toUsable ?~> Messages("dataset.notImported", inboxDataSource.id.name) - oldPrecedenceLayer <- fetchOldPrecedenceLayer(annotation.annotationLayers, dataset) - oldPrecedenceLayerProperties: Option[RedundantTracingProperties] = oldPrecedenceLayer.map( - extractPrecedenceProperties) + tracingStoreClient <- tracingStoreService.clientFor(dataset) + + /* + Note that the tracings have redundant properties, with a precedence logic selecting a layer + from which the values are used. Adding a layer may change this precedence, so the redundant + values need to be copied to the new layer from the layer that had precedence before. Otherwise, those + properties would be masked and lost. + Unfortunately, their history is still lost since the new layer gets only the latest snapshot. + We do this for *every* new layer, since we only later get its ID which determines the actual precedence. + All of this is skipped if existingAnnotationLayers is empty. + */ + oldPrecedenceLayerProperties <- getOldPrecedenceLayerProperties(existingAnnotationLayers, + dataset, + tracingStoreClient) tracing <- params.typ match { case AnnotationLayerType.Skeleton => val skeleton = SkeletonTracingDefaults.createInstance.copy( @@ -279,15 +221,7 @@ class AnnotationService @Inject()( organizationId = Some(dataset._organization), additionalAxes = AdditionalAxis.toProto(dataSource.additionalAxesUnion) ) - val skeletonAdapted = oldPrecedenceLayerProperties.map { p: RedundantTracingProperties => - skeleton.copy( - editPosition = p.editPosition, - editRotation = p.editRotation, - zoomLevel = p.zoomLevel, - userBoundingBoxes = p.userBoundingBoxes, - editPositionAdditionalCoordinates = p.editPositionAdditionalCoordinates - ) - }.getOrElse(skeleton) + val skeletonAdapted = adaptSkeletonTracing(skeleton, oldPrecedenceLayerProperties) Fox.successful(Left(skeletonAdapted)) case AnnotationLayerType.Volume => val autoFallbackLayerName = @@ -303,15 +237,7 @@ class AnnotationService @Inject()( resolutionRestrictions = params.resolutionRestrictions.getOrElse(ResolutionRestrictions.empty), mappingName = params.mappingName ) - volumeTracingAdapted = oldPrecedenceLayerProperties.map { p: RedundantTracingProperties => - volumeTracing.copy( - editPosition = p.editPosition, - editRotation = p.editRotation, - zoomLevel = p.zoomLevel, - userBoundingBoxes = p.userBoundingBoxes, - editPositionAdditionalCoordinates = p.editPositionAdditionalCoordinates - ) - }.getOrElse(volumeTracing) + volumeTracingAdapted = adaptVolumeTracing(volumeTracing, oldPrecedenceLayerProperties) } yield Right(volumeTracingAdapted) } } yield tracing @@ -322,26 +248,25 @@ class AnnotationService @Inject()( allAnnotationLayerParameters: List[AnnotationLayerParameters], existingAnnotationLayers: List[AnnotationLayer])( implicit ctx: DBAccessContext, - mp: MessagesProvider): Fox[List[AnnotationLayer]] = { - - def createAndSaveAnnotationLayer(annotationLayerParameters: AnnotationLayerParameters): Fox[AnnotationLayer] = - for { - client <- tracingStoreService.clientFor(dataset) - tracing <- createTracingForExplorational(annotationId, annotationLayerParameters) - layerName = annotationLayerParameters.name.getOrElse( - AnnotationLayer.defaultNameForType(annotationLayerParameters.typ)) - tracingId <- tracing match { - case Left(skeleton) => client.saveSkeletonTracing(skeleton) - case Right(volume) => client.saveVolumeTracing(volume) - } - } yield - AnnotationLayer(tracingId, - annotationLayerParameters.typ, - layerName, - AnnotationLayerStatistics.zeroedForType(annotationLayerParameters.typ)) - - def createAndSaveAnnotationProto(annotationId: ObjectId, annotationLayers: List[AnnotationLayer]): Fox[Unit] = { - val layersProto = annotationLayers.map { l => + mp: MessagesProvider): Fox[List[AnnotationLayer]] = + for { + tracingStoreClient <- tracingStoreService.clientFor(dataset) + newAnnotationLayers <- Fox.serialCombined(allAnnotationLayerParameters) { annotationLayerParameters => + for { + tracing <- createTracingForExplorational(dataset, annotationLayerParameters, existingAnnotationLayers) + layerName = annotationLayerParameters.name.getOrElse( + AnnotationLayer.defaultNameForType(annotationLayerParameters.typ)) + tracingId <- tracing match { + case Left(skeleton) => tracingStoreClient.saveSkeletonTracing(skeleton) + case Right(volume) => tracingStoreClient.saveVolumeTracing(volume) + } + } yield + AnnotationLayer(tracingId, + annotationLayerParameters.typ, + layerName, + AnnotationLayerStatistics.zeroedForType(annotationLayerParameters.typ)) + } + layersProto = newAnnotationLayers.map { l => AnnotationLayerProto( l.tracingId, l.name, @@ -349,41 +274,9 @@ class AnnotationService @Inject()( ) } //TODO pass right name, description here - val annotationProto = AnnotationProto(name = None, description = None, version = 0L, layers = layersProto) - for { - tracingStoreClient <- tracingStoreService.clientFor(dataset) - _ <- tracingStoreClient.saveAnnotationProto(annotationId, annotationProto) - } yield () - } - - for { - /* - Note that the tracings have redundant properties, with a precedence logic selecting a layer - from which the values are used. Adding a layer may change this precedence, so the redundant - values need to be copied to the new layer from the layer that had precedence before. Otherwise, those - properties would be masked and lost. - Unfortunately, their history is still lost since the new layer gets only the latest snapshot. - We do this for *every* new layer, since we only later get its ID which determines the actual precedence. - All of this is skipped if existingAnnotationLayers is empty. - */ - newAnnotationLayers <- Fox.serialCombined(allAnnotationLayerParameters)(createAndSaveAnnotationLayer) - _ <- createAndSaveAnnotationProto(annotationId, newAnnotationLayers) + annotationProto = AnnotationProto(name = None, description = None, version = 0L, layers = layersProto) + _ <- tracingStoreClient.saveAnnotationProto(annotationId, annotationProto) } yield newAnnotationLayers - } - - /* - If there is more than one tracing, select the one that has precedence for the parameters (they should be identical anyway) - This needs to match the code in NmlWriter’s selectLayerWithPrecedence, though the types are different - */ - private def selectLayerWithPrecedence(annotationLayers: List[AnnotationLayer]): Fox[AnnotationLayer] = { - val skeletonLayers = annotationLayers.filter(_.typ == AnnotationLayerType.Skeleton) - val volumeLayers = annotationLayers.filter(_.typ == AnnotationLayerType.Volume) - if (skeletonLayers.nonEmpty) { - Fox.successful(skeletonLayers.minBy(_.tracingId)) - } else if (volumeLayers.nonEmpty) { - Fox.successful(volumeLayers.minBy(_.tracingId)) - } else Fox.failure("Trying to select precedence layer from empty layer list.") - } def createExplorationalFor(user: User, datasetId: ObjectId, diff --git a/app/models/annotation/nml/NmlWriter.scala b/app/models/annotation/nml/NmlWriter.scala index 4a667dd5f57..1a3058c0d46 100644 --- a/app/models/annotation/nml/NmlWriter.scala +++ b/app/models/annotation/nml/NmlWriter.scala @@ -12,7 +12,7 @@ import com.scalableminds.webknossos.datastore.models.VoxelSize import com.scalableminds.webknossos.datastore.models.annotation.{AnnotationLayerType, FetchedAnnotationLayer} import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeDataZipFormat.VolumeDataZipFormat import com.sun.xml.txw2.output.IndentingXMLStreamWriter -import models.annotation.Annotation +import models.annotation.{Annotation, AnnotationLayerPrecedence} import models.task.Task import models.user.User @@ -37,7 +37,7 @@ case class NmlParameters( editPositionAdditionalCoordinates: Seq[AdditionalCoordinateProto] ) -class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits { +class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits with AnnotationLayerPrecedence { private lazy val outputService = XMLOutputFactory.newInstance() def toNmlStream(name: String, @@ -129,7 +129,7 @@ class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits { datasetName: String, voxelSize: Option[VoxelSize]): Fox[NmlParameters] = for { - parameterSourceAnnotationLayer <- selectLayerWithPrecedence(skeletonLayers, volumeLayers) + parameterSourceAnnotationLayer <- selectLayerWithPrecedenceFetched(skeletonLayers, volumeLayers) nmlParameters = parameterSourceAnnotationLayer.tracing match { case Left(s) => NmlParameters( @@ -168,15 +168,6 @@ class NmlWriter @Inject()(implicit ec: ExecutionContext) extends FoxImplicits { } } yield nmlParameters - // If there is more than one tracing, select the one that has precedence for the parameters (they should be identical anyway) - private def selectLayerWithPrecedence(skeletonLayers: List[FetchedAnnotationLayer], - volumeLayers: List[FetchedAnnotationLayer]): Fox[FetchedAnnotationLayer] = - if (skeletonLayers.nonEmpty) { - Fox.successful(skeletonLayers.minBy(_.tracingId)) - } else if (volumeLayers.nonEmpty) { - Fox.successful(volumeLayers.minBy(_.tracingId)) - } else Fox.failure("annotation.download.noLayers") - private def writeParameters(parameters: NmlParameters)(implicit writer: XMLStreamWriter): Unit = Xml.withinElementSync("parameters") { Xml.withinElementSync("experiment") { diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes index 88bd96335ac..94240d3faad 100644 --- a/conf/webknossos.latest.routes +++ b/conf/webknossos.latest.routes @@ -148,8 +148,6 @@ PATCH /annotations/:typ/:id/editLockedState GET /annotations/:id/info controllers.AnnotationController.infoWithoutType(id: String, timestamp: Option[Long]) PATCH /annotations/:id/downsample controllers.AnnotationController.downsampleWithoutType(id: String, tracingId: String) -PATCH /annotations/:id/addAnnotationLayer controllers.AnnotationController.addAnnotationLayerWithoutType(id: String) -PATCH /annotations/:id/deleteAnnotationLayer controllers.AnnotationController.deleteAnnotationLayerWithoutType(id: String, layerName: String) DELETE /annotations/:id controllers.AnnotationController.cancelWithoutType(id: String) POST /annotations/:id/merge/:mergedTyp/:mergedId controllers.AnnotationController.mergeWithoutType(id: String, mergedTyp: String, mergedId: String) GET /annotations/:id/download controllers.AnnotationIOController.downloadWithoutType(id: String, skeletonVersion: Option[Long], volumeVersion: Option[Long], skipVolumeData: Option[Boolean], volumeDataZipFormat: Option[String]) @@ -158,8 +156,6 @@ PATCH /annotations/addSegmentIndicesToAll GET /annotations/:typ/:id/info controllers.AnnotationController.info(typ: String, id: String, timestamp: Option[Long]) PATCH /annotations/:typ/:id/downsample controllers.AnnotationController.downsample(typ: String, id: String, tracingId: String) -PATCH /annotations/:typ/:id/addAnnotationLayer controllers.AnnotationController.addAnnotationLayer(typ: String, id: String) -PATCH /annotations/:typ/:id/deleteAnnotationLayer controllers.AnnotationController.deleteAnnotationLayer(typ: String, id: String, layerName: String) DELETE /annotations/:typ/:id controllers.AnnotationController.cancel(typ: String, id: String) POST /annotations/:typ/:id/merge/:mergedTyp/:mergedId controllers.AnnotationController.merge(typ: String, id: String, mergedTyp: String, mergedId: String) GET /annotations/:typ/:id/download controllers.AnnotationIOController.download(typ: String, id: String, skeletonVersion: Option[Long], volumeVersion: Option[Long], skipVolumeData: Option[Boolean], volumeDataZipFormat: Option[String]) From 344fd01c44cf2d6e37988d9892fad298a0d4b5da Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 21 Oct 2024 11:49:49 +0200 Subject: [PATCH 109/150] fetch precedence layer from previous version --- .../WKRemoteTracingStoreController.scala | 8 ++++++-- .../AnnotationLayerPrecedence.scala | 12 ++++++++---- app/models/annotation/AnnotationService.scala | 9 +++++++-- conf/webknossos.latest.routes | 2 +- .../TSRemoteWebknossosClient.scala | 5 +++-- .../annotation/TSAnnotationService.scala | 19 ++++++++++--------- 6 files changed, 35 insertions(+), 20 deletions(-) diff --git a/app/controllers/WKRemoteTracingStoreController.scala b/app/controllers/WKRemoteTracingStoreController.scala index 4ce52bdef84..eebbf594f88 100644 --- a/app/controllers/WKRemoteTracingStoreController.scala +++ b/app/controllers/WKRemoteTracingStoreController.scala @@ -175,7 +175,10 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore } } - def createTracing(name: String, key: String, annotationId: String): Action[AnnotationLayerParameters] = + def createTracing(name: String, + key: String, + annotationId: String, + previousVersion: Long): Action[AnnotationLayerParameters] = Action.async(validateJson[AnnotationLayerParameters]) { implicit request => tracingStoreService.validateAccess(name, key) { _ => implicit val ctx: DBAccessContext = GlobalAccessContext @@ -185,7 +188,8 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore dataset <- datasetDAO.findOne(annotation._dataset) tracingEither <- annotationService.createTracingForExplorational(dataset, request.body, - annotation.annotationLayers) + annotation.annotationLayers, + Some(previousVersion)) tracing: GeneratedMessage = tracingEither match { case Left(s: SkeletonTracing) => s case Right(v: VolumeTracing) => v diff --git a/app/models/annotation/AnnotationLayerPrecedence.scala b/app/models/annotation/AnnotationLayerPrecedence.scala index 8d255cae38c..96ac3ec77fd 100644 --- a/app/models/annotation/AnnotationLayerPrecedence.scala +++ b/app/models/annotation/AnnotationLayerPrecedence.scala @@ -20,7 +20,6 @@ import models.dataset.Dataset import scala.concurrent.ExecutionContext // Used to pass duplicate properties when creating a new tracing to avoid masking them. -// Uses the proto-generated geometry classes, hence the full qualifiers. case class RedundantTracingProperties( editPosition: Vec3IntProto, editRotation: Vec3DoubleProto, @@ -71,11 +70,15 @@ trait AnnotationLayerPrecedence { }.getOrElse(volumeTracing) protected def getOldPrecedenceLayerProperties(existingAnnotationLayers: List[AnnotationLayer], + previousVersion: Option[Long], dataset: Dataset, tracingStoreClient: WKRemoteTracingStoreClient)( implicit ec: ExecutionContext): Fox[Option[RedundantTracingProperties]] = for { - oldPrecedenceLayer <- fetchOldPrecedenceLayer(existingAnnotationLayers, dataset, tracingStoreClient) + oldPrecedenceLayer <- fetchOldPrecedenceLayer(existingAnnotationLayers, + previousVersion, + dataset, + tracingStoreClient) oldPrecedenceLayerProperties: Option[RedundantTracingProperties] = oldPrecedenceLayer.map( extractPrecedenceProperties) } yield oldPrecedenceLayerProperties @@ -102,6 +105,7 @@ trait AnnotationLayerPrecedence { } private def fetchOldPrecedenceLayer(existingAnnotationLayers: List[AnnotationLayer], + previousVersion: Option[Long], dataset: Dataset, tracingStoreClient: WKRemoteTracingStoreClient)( implicit ec: ExecutionContext): Fox[Option[FetchedAnnotationLayer]] = @@ -110,10 +114,10 @@ trait AnnotationLayerPrecedence { for { oldPrecedenceLayer <- selectLayerWithPrecedence(existingAnnotationLayers) oldPrecedenceLayerFetched <- if (oldPrecedenceLayer.typ == AnnotationLayerType.Skeleton) - tracingStoreClient.getSkeletonTracing(oldPrecedenceLayer, None) + tracingStoreClient.getSkeletonTracing(oldPrecedenceLayer, previousVersion) else tracingStoreClient.getVolumeTracing(oldPrecedenceLayer, - None, + previousVersion, skipVolumeData = true, volumeDataZipFormat = VolumeDataZipFormat.wkw, dataset.voxelSize) diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index 7c22ca76ec0..bda33814f37 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -167,7 +167,8 @@ class AnnotationService @Inject()( def createTracingForExplorational(dataset: Dataset, params: AnnotationLayerParameters, - existingAnnotationLayers: List[AnnotationLayer])( + existingAnnotationLayers: List[AnnotationLayer], + previousVersion: Option[Long])( implicit ctx: DBAccessContext, mp: MessagesProvider): Fox[Either[SkeletonTracing, VolumeTracing]] = { @@ -211,6 +212,7 @@ class AnnotationService @Inject()( All of this is skipped if existingAnnotationLayers is empty. */ oldPrecedenceLayerProperties <- getOldPrecedenceLayerProperties(existingAnnotationLayers, + previousVersion, dataset, tracingStoreClient) tracing <- params.typ match { @@ -253,7 +255,10 @@ class AnnotationService @Inject()( tracingStoreClient <- tracingStoreService.clientFor(dataset) newAnnotationLayers <- Fox.serialCombined(allAnnotationLayerParameters) { annotationLayerParameters => for { - tracing <- createTracingForExplorational(dataset, annotationLayerParameters, existingAnnotationLayers) + tracing <- createTracingForExplorational(dataset, + annotationLayerParameters, + existingAnnotationLayers, + previousVersion = None) layerName = annotationLayerParameters.name.getOrElse( AnnotationLayer.defaultNameForType(annotationLayerParameters.typ)) tracingId <- tracing match { diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes index 94240d3faad..a1ace976868 100644 --- a/conf/webknossos.latest.routes +++ b/conf/webknossos.latest.routes @@ -128,7 +128,7 @@ GET /tracingstores/:name/dataSource GET /tracingstores/:name/dataSourceId controllers.WKRemoteTracingStoreController.dataSourceIdForTracing(name: String, key: String, tracingId: String) GET /tracingstores/:name/annotationId controllers.WKRemoteTracingStoreController.annotationIdForTracing(name: String, key: String, tracingId: String) GET /tracingstores/:name/dataStoreUri/:datasetName controllers.WKRemoteTracingStoreController.dataStoreUriForDataset(name: String, key: String, organizationId: Option[String], datasetName: String) -POST /tracingstores/:name/createTracing controllers.WKRemoteTracingStoreController.createTracing(name: String, key: String, annotationId: String) +POST /tracingstores/:name/createTracing controllers.WKRemoteTracingStoreController.createTracing(name: String, key: String, annotationId: String, previousVersion: Long) # User access tokens for datastore authentication POST /userToken/generate controllers.UserTokenController.generateTokenForDataStore() diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala index 43f73e1a87f..52ddf6f03f4 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala @@ -17,7 +17,6 @@ import com.scalableminds.webknossos.datastore.services.{ UserAccessRequest } import com.scalableminds.webknossos.tracingstore.annotation.AnnotationLayerParameters -import com.scalableminds.webknossos.tracingstore.tracings.TracingType import com.typesafe.scalalogging.LazyLogging import play.api.inject.ApplicationLifecycle import play.api.libs.json.{JsObject, Json, OFormat} @@ -101,9 +100,11 @@ class TSRemoteWebknossosClient @Inject()( .postJson(annotationLayers) def createTracingFor(annotationId: String, - layerParameters: AnnotationLayerParameters): Fox[Either[SkeletonTracing, VolumeTracing]] = { + layerParameters: AnnotationLayerParameters, + previousVersion: Long): Fox[Either[SkeletonTracing, VolumeTracing]] = { val req = rpc(s"$webknossosUri/api/tracingstores/$tracingStoreName/createTracing") .addQueryString("annotationId" -> annotationId) + .addQueryString("previousVersion" -> previousVersion.toString) // used for fetching old precedence layers .addQueryString("key" -> tracingStoreKey) layerParameters.typ match { case AnnotationLayerType.Volume => diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index a8911ed7e69..5a9b87a4c7c 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -108,7 +108,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss for { updated <- updateAction match { case a: AddLayerAnnotationUpdateAction => - addLayer(annotationId, annotationWithTracings, a) + addLayer(annotationId, annotationWithTracings, a, targetVersion) case a: DeleteLayerAnnotationUpdateAction => Fox.successful(annotationWithTracings.deleteTracing(a)) case a: UpdateLayerMetadataAnnotationUpdateAction => @@ -134,14 +134,15 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } } yield updated - private def addLayer( - annotationId: String, - annotationWithTracings: AnnotationWithTracings, - action: AddLayerAnnotationUpdateAction)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = + private def addLayer(annotationId: String, + annotationWithTracings: AnnotationWithTracings, + action: AddLayerAnnotationUpdateAction, + targetVersion: Long)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { - _ <- Fox.successful(()) - tracingId <- action.tracingId ?~> "add layer action has no tracingId" - tracing <- remoteWebknossosClient.createTracingFor(annotationId, action.layerParameters) + tracingId <- action.tracingId.toFox ?~> "add layer action has no tracingId" + tracing <- remoteWebknossosClient.createTracingFor(annotationId, + action.layerParameters, + previousVersion = targetVersion - 1) updated = annotationWithTracings.addLayer(action, tracingId, tracing) } yield updated @@ -436,7 +437,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss for { updated <- updateIter(Some(annotation.withNewUpdaters(annotation.version, targetVersion)), updates) updatedWithNewVerson = updated.withVersion(targetVersion) - _ = logger.info(s"flushing v${targetVersion}, with ${updated.skeletonStats}") + _ = logger.info(s"flushing v$targetVersion, with ${updated.skeletonStats}") _ <- updatedWithNewVerson.flushBufferedUpdates() _ <- flushUpdatedTracings(updatedWithNewVerson) _ <- flushAnnotationInfo(annotationId, updatedWithNewVerson) From 5f13149afce4bc624e0b099f2697d516723b3df7 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 21 Oct 2024 11:51:47 +0200 Subject: [PATCH 110/150] isolate add layer update actions --- test/backend/UpdateGroupHandlingUnitTestSuite.scala | 2 +- .../annotation/TSAnnotationService.scala | 2 +- .../annotation/UpdateGroupHandling.scala | 13 +++++++------ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/test/backend/UpdateGroupHandlingUnitTestSuite.scala b/test/backend/UpdateGroupHandlingUnitTestSuite.scala index bdc06c79f36..dec49103cd1 100644 --- a/test/backend/UpdateGroupHandlingUnitTestSuite.scala +++ b/test/backend/UpdateGroupHandlingUnitTestSuite.scala @@ -29,7 +29,7 @@ class UpdateGroupHandlingUnitTestSuite extends PlaySpec with UpdateGroupHandling MergeTreeSkeletonAction(sourceId = 2, targetId = 3, actionTracingId = Dummies.tracingId) )), ) - val res = regroupByRevertActions(updateGroupsBefore) + val res = regroupByIsolationSensitiveActions(updateGroupsBefore) assert(res.length == 3) assert(res(1)._2.length == 1) assert(res(1)._1 == 6L) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 5a9b87a4c7c..6c8a91c6e80 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -272,7 +272,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss requestAll: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = for { updateGroupsAsSaved <- findPendingUpdates(annotationId, annotation.version, targetVersion) ?~> "findPendingUpdates.failed" - updatesGroupsRegrouped = regroupByRevertActions(updateGroupsAsSaved) + updatesGroupsRegrouped = regroupByIsolationSensitiveActions(updateGroupsAsSaved) updatesFlat = updatesGroupsRegrouped.flatMap(_._2) annotationWithTracings <- findTracingsForUpdates(annotation, updatesFlat, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateGroupHandling.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateGroupHandling.scala index 6b8f4cf0e04..6893d1bcdc5 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateGroupHandling.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateGroupHandling.scala @@ -4,12 +4,12 @@ import collections.SequenceUtils trait UpdateGroupHandling { - def regroupByRevertActions( + def regroupByIsolationSensitiveActions( updateActionGroupsWithVersions: List[(Long, List[UpdateAction])]): List[(Long, List[UpdateAction])] = { val splitGroupLists: List[List[(Long, List[UpdateAction])]] = SequenceUtils.splitAndIsolate(updateActionGroupsWithVersions.reverse)(actionGroup => - actionGroup._2.exists(updateAction => isRevertAction(updateAction))) - // TODO assert that the groups that contain revert actions contain nothing else + actionGroup._2.exists(updateAction => isIsolationSensitiveAction(updateAction))) + // TODO assert that the *groups* that contain revert actions contain nothing else // TODO test this splitGroupLists.flatMap { groupsToConcatenate: List[(Long, List[UpdateAction])] => @@ -24,8 +24,9 @@ trait UpdateGroupHandling { targetVersionOpt.map(targetVersion => (targetVersion, updates)) } - private def isRevertAction(a: UpdateAction): Boolean = a match { - case _: RevertToVersionUpdateAction => true - case _ => false + private def isIsolationSensitiveAction(a: UpdateAction): Boolean = a match { + case _: RevertToVersionUpdateAction => true + case _: AddLayerAnnotationUpdateAction => true + case _ => false } } From d5b538649cae31d08b9274e2290ec911857ca400 Mon Sep 17 00:00:00 2001 From: MichaelBuessemeyer <39529669+MichaelBuessemeyer@users.noreply.github.com> Date: Mon, 21 Oct 2024 16:32:26 +0200 Subject: [PATCH 111/150] Unified annotation restore (#8136) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * WIP: unified version restore frontend * WIP unified version restore * add segmentation layer name to relevant version restore entries & save correct annotation version in store when fetching an annotation version * small cleanup * fix compiler warning, remove comment --------- Co-authored-by: Michael Büßemeyer Co-authored-by: Florian M --- frontend/javascripts/admin/admin_rest_api.ts | 30 +-- frontend/javascripts/oxalis/api/api_latest.ts | 5 +- frontend/javascripts/oxalis/controller.tsx | 12 +- frontend/javascripts/oxalis/default_state.ts | 1 + frontend/javascripts/oxalis/model.ts | 5 +- .../model/accessors/annotation_accessor.ts | 72 +++---- .../oxalis/model/actions/save_actions.ts | 12 +- .../model/bucket_data_handling/pushqueue.ts | 8 +- .../javascripts/oxalis/model/data_layer.ts | 9 +- .../compaction/compact_toggle_actions.ts | 1 - .../compaction/compact_update_actions.ts | 1 - .../oxalis/model/reducers/save_reducer.ts | 40 +--- .../oxalis/model/sagas/proofread_saga.ts | 9 +- .../oxalis/model/sagas/save_saga.ts | 20 +- .../oxalis/model/sagas/save_saga_constants.ts | 6 +- .../oxalis/model/sagas/volumetracing_saga.tsx | 6 +- .../oxalis/model_initialization.ts | 15 +- frontend/javascripts/oxalis/store.ts | 7 +- .../oxalis/view/action-bar/save_button.tsx | 1 - .../dataset_info_tab_view.tsx | 7 +- .../right-border-tabs/skeleton_tab_view.tsx | 8 +- .../javascripts/oxalis/view/version_entry.tsx | 134 +++++++++---- .../javascripts/oxalis/view/version_list.tsx | 83 +++----- .../javascripts/oxalis/view/version_view.tsx | 187 ++++++------------ frontend/javascripts/types/api_flow_types.ts | 1 + .../annotation/AnnotationLayerType.scala | 2 + 26 files changed, 277 insertions(+), 405 deletions(-) diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts index 28204b26da5..aa2e4b95e8e 100644 --- a/frontend/javascripts/admin/admin_rest_api.ts +++ b/frontend/javascripts/admin/admin_rest_api.ts @@ -83,7 +83,6 @@ import type { import type { NewTask, TaskCreationResponseContainer } from "admin/task/task_create_bulk_view"; import type { QueryObject } from "admin/task/task_search_form"; import { V3 } from "libs/mjs"; -import type { Versions } from "oxalis/view/version_view"; import { enforceValidatedDatasetViewConfiguration } from "types/schemas/dataset_view_configuration_defaults"; import { parseProtoListOfLong, @@ -97,7 +96,6 @@ import Toast from "libs/toast"; import * as Utils from "libs/utils"; import messages from "messages"; import window, { location } from "libs/window"; -import type { SaveQueueType } from "oxalis/model/actions/save_actions"; import type { DatasourceConfiguration } from "types/schemas/datasource.types"; import { doWithToken } from "./api/token"; import type BoundingBox from "oxalis/model/bucket_data_handling/bounding_box"; @@ -841,12 +839,12 @@ export function createExplorational( export async function getTracingsForAnnotation( annotation: APIAnnotation, - versions: Versions = {}, + version: number | null | undefined, ): Promise> { const skeletonLayers = annotation.annotationLayers.filter((layer) => layer.typ === "Skeleton"); const fullAnnotationLayers = await Promise.all( annotation.annotationLayers.map((layer) => - getTracingForAnnotationType(annotation, layer, versions), + getTracingForAnnotationType(annotation, layer, version), ), ); @@ -871,27 +869,12 @@ export async function acquireAnnotationMutex( return { canEdit, blockedByUser }; } -function extractVersion( - versions: Versions, - tracingId: string, - typ: "Volume" | "Skeleton", -): number | null | undefined { - if (typ === "Skeleton") { - return versions.skeleton; - } else if (versions.volumes != null) { - return versions.volumes[tracingId]; - } - - return null; -} - export async function getTracingForAnnotationType( annotation: APIAnnotation, annotationLayerDescriptor: AnnotationLayerDescriptor, - versions: Versions = {}, + version?: number | null | undefined, // TODO: Use this parameter ): Promise { const { tracingId, typ } = annotationLayerDescriptor; - const version = extractVersion(versions, tracingId, typ); const tracingType = typ.toLowerCase() as "skeleton" | "volume"; const possibleVersionString = version != null ? `&version=${version}` : ""; const tracingArrayBuffer = await doWithToken((token) => @@ -1042,16 +1025,17 @@ export async function downloadAnnotation( annotationId: string, annotationType: APIAnnotationType, showVolumeFallbackDownloadWarning: boolean = false, - versions: Versions = {}, + _version: number | null | undefined = null, downloadFileFormat: "zarr3" | "wkw" | "nml" = "wkw", includeVolumeData: boolean = true, ) { const searchParams = new URLSearchParams(); - Object.entries(versions).forEach(([key, val]) => { + // TODO: Use the version parameter + /*Object.entries(versions).forEach(([key, val]) => { if (val != null) { searchParams.append(`${key}Version`, val.toString()); } - }); + });*/ if (includeVolumeData && showVolumeFallbackDownloadWarning) { Toast.info(messages["annotation.no_fallback_data_included"], { diff --git a/frontend/javascripts/oxalis/api/api_latest.ts b/frontend/javascripts/oxalis/api/api_latest.ts index 404d508f660..b1d8d606574 100644 --- a/frontend/javascripts/oxalis/api/api_latest.ts +++ b/frontend/javascripts/oxalis/api/api_latest.ts @@ -7,7 +7,6 @@ import { getConstructorForElementClass } from "oxalis/model/bucket_data_handling import { type APICompoundType, APICompoundTypeEnum, type ElementClass } from "types/api_flow_types"; import { InputKeyboardNoLoop } from "libs/input"; import { M4x4, type Matrix4x4, V3, type Vector16 } from "libs/mjs"; -import type { Versions } from "oxalis/view/version_view"; import { addTreesAndGroupsAction, setActiveNodeAction, @@ -1115,7 +1114,7 @@ class TracingApi { newMaybeCompoundType: APICompoundType | null, newAnnotationId: string, newControlMode: ControlMode, - versions?: Versions, + version?: number | undefined | null, keepUrlState: boolean = false, ) { if (newControlMode === ControlModeEnum.VIEW) @@ -1134,7 +1133,7 @@ class TracingApi { type: newControlMode, }, false, - versions, + version, ); Store.dispatch(discardSaveQueuesAction()); Store.dispatch(wkReadyAction()); diff --git a/frontend/javascripts/oxalis/controller.tsx b/frontend/javascripts/oxalis/controller.tsx index 0912a7babe9..537977d8e9d 100644 --- a/frontend/javascripts/oxalis/controller.tsx +++ b/frontend/javascripts/oxalis/controller.tsx @@ -90,14 +90,12 @@ class Controller extends React.PureComponent { tryFetchingModel() { this.props.setControllerStatus("loading"); // Preview a working annotation version if the showVersionRestore URL parameter is supplied - const versions = Utils.hasUrlParam("showVersionRestore") - ? { - skeleton: Utils.hasUrlParam("skeletonVersion") - ? Number.parseInt(Utils.getUrlParamValue("skeletonVersion")) - : 1, - } + const version = Utils.hasUrlParam("showVersionRestore") + ? Utils.hasUrlParam("version") + ? Number.parseInt(Utils.getUrlParamValue("version")) + : 1 : undefined; - Model.fetch(this.props.initialMaybeCompoundType, this.props.initialCommandType, true, versions) + Model.fetch(this.props.initialMaybeCompoundType, this.props.initialCommandType, true, version) .then(() => this.modelFetchDone()) .catch((error) => { this.props.setControllerStatus("failedLoading"); diff --git a/frontend/javascripts/oxalis/default_state.ts b/frontend/javascripts/oxalis/default_state.ts index 6124b8cdfa7..8f42b637ef8 100644 --- a/frontend/javascripts/oxalis/default_state.ts +++ b/frontend/javascripts/oxalis/default_state.ts @@ -177,6 +177,7 @@ const defaultState: OxalisState = { othersMayEdit: false, blockedByUser: null, annotationLayers: [], + version: 0, }, save: { queue: [], diff --git a/frontend/javascripts/oxalis/model.ts b/frontend/javascripts/oxalis/model.ts index f3c7e63e7bf..07f84588d31 100644 --- a/frontend/javascripts/oxalis/model.ts +++ b/frontend/javascripts/oxalis/model.ts @@ -1,6 +1,5 @@ import _ from "lodash"; import type { Vector3 } from "oxalis/constants"; -import type { Versions } from "oxalis/view/version_view"; import { getActiveSegmentationTracingLayer } from "oxalis/model/accessors/volumetracing_accessor"; import { getActiveMagIndexForLayer } from "oxalis/model/accessors/flycam_accessor"; import { @@ -32,14 +31,14 @@ export class OxalisModel { initialMaybeCompoundType: APICompoundType | null, initialCommandType: TraceOrViewCommand, initialFetch: boolean, - versions?: Versions, + version?: number | undefined | null, ) { try { const initializationInformation = await initialize( initialMaybeCompoundType, initialCommandType, initialFetch, - versions, + version, ); if (initializationInformation) { diff --git a/frontend/javascripts/oxalis/model/accessors/annotation_accessor.ts b/frontend/javascripts/oxalis/model/accessors/annotation_accessor.ts index 1949bbadd23..092a5657d80 100644 --- a/frontend/javascripts/oxalis/model/accessors/annotation_accessor.ts +++ b/frontend/javascripts/oxalis/model/accessors/annotation_accessor.ts @@ -1,6 +1,5 @@ import _ from "lodash"; import type { OxalisState, Tracing } from "oxalis/store"; -import { getVolumeTracingById } from "./volumetracing_accessor"; import type { APIAnnotationInfo } from "types/api_flow_types"; import type { EmptyObject } from "types/globals"; @@ -47,60 +46,35 @@ type TracingStatsHelper = { // biome-ignore lint/complexity/noBannedTypes: {} should be avoided actually export type CombinedTracingStats = (SkeletonTracingStats | {}) & (VolumeTracingStats | {}); -export function getStats( - tracing: Tracing, - saveQueueType: "skeleton" | "volume" | "mapping", - tracingId: string, -): TracingStats | null { - switch (saveQueueType) { - case "skeleton": { - if (!tracing.skeleton) { - return null; - } - const trees = tracing.skeleton.trees; - return { - treeCount: _.size(trees), - nodeCount: _.reduce(trees, (sum, tree) => sum + tree.nodes.size(), 0), - edgeCount: _.reduce(trees, (sum, tree) => sum + tree.edges.size(), 0), - branchPointCount: _.reduce(trees, (sum, tree) => sum + _.size(tree.branchPoints), 0), - }; - } - case "volume": { - const volumeTracing = getVolumeTracingById(tracing, tracingId); - return { - segmentCount: volumeTracing.segments.size(), - }; - } - default: - return null; +export function getStats(tracing: Tracing): CombinedTracingStats { + const { skeleton, volumes } = tracing; + let totalSegmentCount = 0; + for (const volumeTracing of volumes) { + totalSegmentCount += volumeTracing.segments.size(); } -} - -export function getCombinedStats(tracing: Tracing): CombinedTracingStats { - const aggregatedStats: TracingStatsHelper = {}; - - if (tracing.skeleton) { - const skeletonStats = getStats(tracing, "skeleton", tracing.skeleton.tracingId); - if (skeletonStats && "treeCount" in skeletonStats) { - const { treeCount, nodeCount, edgeCount, branchPointCount } = skeletonStats; - aggregatedStats.treeCount = treeCount; - aggregatedStats.nodeCount = nodeCount; - aggregatedStats.edgeCount = edgeCount; - aggregatedStats.branchPointCount = branchPointCount; - } + let stats: TracingStats = { + segmentCount: totalSegmentCount, + }; + if (skeleton) { + stats = { + ...stats, + treeCount: _.size(skeleton.trees), + nodeCount: _.reduce(skeleton.trees, (sum, tree) => sum + tree.nodes.size(), 0), + edgeCount: _.reduce(skeleton.trees, (sum, tree) => sum + tree.edges.size(), 0), + branchPointCount: _.reduce(skeleton.trees, (sum, tree) => sum + _.size(tree.branchPoints), 0), + }; } + return stats; +} +export function getCreationTimestamp(tracing: Tracing) { + let timestamp = tracing.skeleton?.createdTimestamp; for (const volumeTracing of tracing.volumes) { - const volumeStats = getStats(tracing, "volume", volumeTracing.tracingId); - if (volumeStats && "segmentCount" in volumeStats) { - if (aggregatedStats.segmentCount == null) { - aggregatedStats.segmentCount = 0; - } - aggregatedStats.segmentCount += volumeStats.segmentCount; + if (!timestamp || volumeTracing.createdTimestamp < timestamp) { + timestamp = volumeTracing.createdTimestamp; } } - - return aggregatedStats; + return timestamp || 0; } export function getCombinedStatsFromServerAnnotation( diff --git a/frontend/javascripts/oxalis/model/actions/save_actions.ts b/frontend/javascripts/oxalis/model/actions/save_actions.ts index 42d8e47747c..870ba1f730f 100644 --- a/frontend/javascripts/oxalis/model/actions/save_actions.ts +++ b/frontend/javascripts/oxalis/model/actions/save_actions.ts @@ -30,16 +30,14 @@ export type SaveAction = export const pushSaveQueueTransaction = ( items: Array, - saveQueueType: SaveQueueType, tracingId: string, transactionId: string = getUid(), ) => ({ type: "PUSH_SAVE_QUEUE_TRANSACTION", items, - saveQueueType, - tracingId, transactionId, + tracingId, }) as const; export const saveNowAction = () => @@ -70,16 +68,10 @@ export const setLastSaveTimestampAction = () => timestamp: Date.now(), }) as const; -export const setVersionNumberAction = ( - version: number, - saveQueueType: SaveQueueType, - tracingId: string, -) => +export const setVersionNumberAction = (version: number) => ({ type: "SET_VERSION_NUMBER", version, - saveQueueType, - tracingId, }) as const; export const undoAction = (callback?: () => void) => diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/pushqueue.ts b/frontend/javascripts/oxalis/model/bucket_data_handling/pushqueue.ts index 894261633a5..c5e1294bf6e 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/pushqueue.ts +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/pushqueue.ts @@ -17,6 +17,7 @@ const PUSH_DEBOUNCE_TIME = 1000; class PushQueue { cube: DataCube; + tracingId: string; // The pendingBuckets contains all buckets that should be: // - snapshotted, @@ -41,8 +42,9 @@ class PushQueue { // transaction. private waitTimeStartTimeStamp: number | null = null; - constructor(cube: DataCube) { + constructor(cube: DataCube, tracingId: string) { this.cube = cube; + this.tracingId = tracingId; this.pendingBuckets = new Set(); } @@ -131,7 +133,7 @@ class PushQueue { push = createDebouncedAbortableParameterlessCallable(this.pushImpl, PUSH_DEBOUNCE_TIME, this); - async pushTransaction(batch: Array): Promise { + private async pushTransaction(batch: Array): Promise { /* * Create a transaction from the batch and push it into the save queue. */ @@ -152,7 +154,7 @@ class PushQueue { const items = await this.fifoResolver.orderedWaitFor( createCompressedUpdateBucketActions(batch), ); - Store.dispatch(pushSaveQueueTransaction(items, "volume", this.cube.layerName)); + Store.dispatch(pushSaveQueueTransaction(items, this.tracingId, this.cube.layerName)); this.compressingBucketCount -= batch.length; } catch (error) { diff --git a/frontend/javascripts/oxalis/model/data_layer.ts b/frontend/javascripts/oxalis/model/data_layer.ts index d5b3053c543..5ceb62998aa 100644 --- a/frontend/javascripts/oxalis/model/data_layer.ts +++ b/frontend/javascripts/oxalis/model/data_layer.ts @@ -21,7 +21,12 @@ class DataLayer { fallbackLayerInfo: DataLayerType | null | undefined; isSegmentation: boolean; - constructor(layerInfo: DataLayerType, textureWidth: number, dataTextureCount: number) { + constructor( + layerInfo: DataLayerType, + textureWidth: number, + dataTextureCount: number, + tracingId: string, + ) { this.name = layerInfo.name; this.fallbackLayer = "fallbackLayer" in layerInfo && layerInfo.fallbackLayer != null @@ -46,7 +51,7 @@ class DataLayer { this.name, ); this.pullQueue = new PullQueue(this.cube, layerInfo.name, dataset.dataStore); - this.pushQueue = new PushQueue(this.cube); + this.pushQueue = new PushQueue(this.cube, tracingId); this.cube.initializeWithQueues(this.pullQueue, this.pushQueue); if (this.isSegmentation) { diff --git a/frontend/javascripts/oxalis/model/helpers/compaction/compact_toggle_actions.ts b/frontend/javascripts/oxalis/model/helpers/compaction/compact_toggle_actions.ts index fd989003fab..cc7ee5af199 100644 --- a/frontend/javascripts/oxalis/model/helpers/compaction/compact_toggle_actions.ts +++ b/frontend/javascripts/oxalis/model/helpers/compaction/compact_toggle_actions.ts @@ -7,7 +7,6 @@ import _ from "lodash"; import type { SkeletonTracing, Tree, TreeGroup, TreeMap, VolumeTracing } from "oxalis/store"; import type { UpdateAction, - UpdateActionWithTracingId, UpdateTreeVisibilityUpdateAction, } from "oxalis/model/sagas/update_actions"; import { updateTreeGroupVisibility, updateTreeVisibility } from "oxalis/model/sagas/update_actions"; diff --git a/frontend/javascripts/oxalis/model/helpers/compaction/compact_update_actions.ts b/frontend/javascripts/oxalis/model/helpers/compaction/compact_update_actions.ts index c9e714a8229..b16e490e5e8 100644 --- a/frontend/javascripts/oxalis/model/helpers/compaction/compact_update_actions.ts +++ b/frontend/javascripts/oxalis/model/helpers/compaction/compact_update_actions.ts @@ -7,7 +7,6 @@ import type { DeleteNodeUpdateAction, DeleteTreeUpdateAction, UpdateAction, - UpdateActionWithTracingId, } from "oxalis/model/sagas/update_actions"; import { moveTreeComponent } from "oxalis/model/sagas/update_actions"; import compactToggleActions from "oxalis/model/helpers/compaction/compact_toggle_actions"; diff --git a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts index 1fab6d3f403..06fb2975175 100644 --- a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts +++ b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts @@ -4,13 +4,9 @@ import type { Action } from "oxalis/model/actions/actions"; import type { OxalisState, SaveState } from "oxalis/store"; import type { SetVersionNumberAction } from "oxalis/model/actions/save_actions"; import { getActionLog } from "oxalis/model/helpers/action_logger_middleware"; -import { getStats } from "oxalis/model/accessors/annotation_accessor"; +import { type CombinedTracingStats, getStats } from "oxalis/model/accessors/annotation_accessor"; import { MAXIMUM_ACTION_COUNT_PER_BATCH } from "oxalis/model/sagas/save_saga_constants"; -import { updateKey2 } from "oxalis/model/helpers/deep_update"; -import { - updateEditableMapping, - updateVolumeTracing, -} from "oxalis/model/reducers/volumetracing_reducer_helpers"; +import { updateKey, updateKey2 } from "oxalis/model/helpers/deep_update"; import Date from "libs/date"; import type { UpdateAction, UpdateActionWithTracingId } from "../sagas/update_actions"; @@ -31,21 +27,9 @@ export function getTotalSaveQueueLength(queueObj: SaveState["queue"]) { } function updateVersion(state: OxalisState, action: SetVersionNumberAction) { - if (action.saveQueueType === "skeleton" && state.tracing.skeleton != null) { - return updateKey2(state, "tracing", "skeleton", { - version: action.version, - }); - } else if (action.saveQueueType === "volume") { - return updateVolumeTracing(state, action.tracingId, { - version: action.version, - }); - } else if (action.saveQueueType === "mapping") { - /*return updateEditableMapping(state, action.tracingId, { - version: action.version, - });*/ - } - - return state; + return updateKey(state, "tracing", { + version: action.version, + }); } function SaveReducer(state: OxalisState, action: Action): OxalisState { @@ -55,25 +39,18 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState { // update actions. const dispatchedAction = action; const { items, transactionId } = dispatchedAction; - if (items.length === 0) { - return state; - } - // Only report tracing statistics, if a "real" update to the tracing happened - const stats = _.some( + const stats: CombinedTracingStats | null = _.some( dispatchedAction.items, (ua) => ua.name !== "updateSkeletonTracing" && ua.name !== "updateVolumeTracing", ) - ? getStats(state.tracing, dispatchedAction.saveQueueType, dispatchedAction.tracingId) + ? getStats(state.tracing) : null; const { activeUser } = state; if (activeUser == null) { throw new Error("Tried to save something even though user is not logged in."); } - const updateActionChunks = _.chunk( - items, - MAXIMUM_ACTION_COUNT_PER_BATCH[dispatchedAction.saveQueueType], - ); + const updateActionChunks = _.chunk(items, MAXIMUM_ACTION_COUNT_PER_BATCH); const transactionGroupCount = updateActionChunks.length; const actionLogInfo = JSON.stringify(getActionLog().slice(-10)); @@ -99,7 +76,6 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState { // caught by the following check. If the bug appears again, we can investigate with more // details thanks to airbrake. if ( - dispatchedAction.saveQueueType === "skeleton" && oldQueue.length > 0 && newQueue.length > 0 && newQueue.at(-1)?.actions.some((action) => NOT_IDEMPOTENT_ACTIONS.includes(action.name)) && diff --git a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts index de523f53795..95f6fa3ad37 100644 --- a/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/proofread_saga.ts @@ -268,6 +268,7 @@ function* createEditableMapping(): Saga { // Save before making the mapping editable to make sure the correct mapping is activated in the backend yield* call([Model, Model.ensureSavedState]); // Get volume tracing again to make sure the version is up to date + const tracing = yield* select((state) => state.tracing); const upToDateVolumeTracing = yield* select((state) => getActiveSegmentationTracing(state)); if (upToDateVolumeTracing == null) { throw new Error("No active segmentation tracing layer. Cannot create editble mapping."); @@ -277,7 +278,7 @@ function* createEditableMapping(): Saga { const layerName = volumeTracingId; const serverEditableMapping = yield* call(makeMappingEditable, tracingStoreUrl, volumeTracingId); // The server increments the volume tracing's version by 1 when switching the mapping to an editable one - yield* put(setVersionNumberAction(upToDateVolumeTracing.version + 1, "volume", volumeTracingId)); + yield* put(setVersionNumberAction(tracing.version + 1)); yield* put(setMappingNameAction(layerName, volumeTracingId, "HDF5")); yield* put(setHasEditableMappingAction()); yield* put(initializeEditableMappingAction(serverEditableMapping)); @@ -453,7 +454,7 @@ function* handleSkeletonProofreadingAction(action: Action): Saga { return; } - yield* put(pushSaveQueueTransaction(items, "mapping", volumeTracingId)); + yield* put(pushSaveQueueTransaction(items, volumeTracingId)); yield* call([Model, Model.ensureSavedState]); if (action.type === "MIN_CUT_AGGLOMERATE_WITH_NODE_IDS" || action.type === "DELETE_EDGE") { @@ -781,7 +782,7 @@ function* handleProofreadMergeOrMinCut(action: Action) { return; } - yield* put(pushSaveQueueTransaction(items, "mapping", volumeTracingId)); + yield* put(pushSaveQueueTransaction(items, volumeTracingId)); yield* call([Model, Model.ensureSavedState]); if (action.type === "MIN_CUT_AGGLOMERATE") { @@ -942,7 +943,7 @@ function* handleProofreadCutFromNeighbors(action: Action) { return; } - yield* put(pushSaveQueueTransaction(items, "mapping", volumeTracingId)); + yield* put(pushSaveQueueTransaction(items, volumeTracingId)); yield* call([Model, Model.ensureSavedState]); // Now that the changes are saved, we can split the mapping locally (because it requires diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts index d94f8851f83..0311b19de8d 100644 --- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts @@ -165,10 +165,11 @@ export function* sendSaveRequestToServer(): Saga { const fullSaveQueue = yield* select((state) => state.save.queue); const saveQueue = sliceAppropriateBatchCount(fullSaveQueue); let compactedSaveQueue = compactSaveQueue(saveQueue); + const tracing = yield* select((state) => state.tracing); const tracings = yield* select((state) => _.compact([state.tracing.skeleton, ...state.tracing.volumes]), ); - const version = _.max(tracings.map((t) => t.version)) || 0; + const version = _.max(tracings.map((t) => t.version).concat([tracing.version])) || 0; const annotationId = yield* select((state) => state.tracing.annotationId); const tracingStoreUrl = yield* select((state) => state.tracing.tracingStore.url); let versionIncrement; @@ -202,11 +203,7 @@ export function* sendSaveRequestToServer(): Saga { ); } - for (const tracing of tracings) { - yield* put( - setVersionNumberAction(version + versionIncrement, tracing.type, tracing.tracingId), - ); - } + yield* put(setVersionNumberAction(version + versionIncrement)); yield* put(setLastSaveTimestampAction()); yield* put(shiftSaveQueueAction(saveQueue.length)); @@ -446,7 +443,7 @@ export function* setupSavingForTracingType( ); if (items.length > 0) { - yield* put(pushSaveQueueTransaction(items, saveQueueType, tracingId)); + yield* put(pushSaveQueueTransaction(items, tracingId)); } prevTracing = tracing; @@ -502,14 +499,7 @@ function* watchForSaveConflicts() { // old reference to tracing might be outdated now due to the // immutability. const versionOnClient = yield* select((state) => { - if (tracing.type === "volume") { - return getVolumeTracingById(state.tracing, tracing.tracingId).version; - } - const { skeleton } = state.tracing; - if (skeleton == null) { - throw new Error("Skeleton must exist at this point."); - } - return skeleton.version; + return state.tracing.version; }); const toastKey = `save_conflicts_warning_${tracing.tracingId}`; diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga_constants.ts b/frontend/javascripts/oxalis/model/sagas/save_saga_constants.ts index 87ace921de4..c9aa2351d4c 100644 --- a/frontend/javascripts/oxalis/model/sagas/save_saga_constants.ts +++ b/frontend/javascripts/oxalis/model/sagas/save_saga_constants.ts @@ -11,11 +11,7 @@ export const UNDO_HISTORY_SIZE = 20; export const SETTINGS_RETRY_DELAY = 15 * 1000; export const SETTINGS_MAX_RETRY_COUNT = 20; // 20 * 15s == 5m -export const MAXIMUM_ACTION_COUNT_PER_BATCH = { - skeleton: 5000, - volume: 1000, // Since volume saving is slower, use a lower value here. - mapping: Number.POSITIVE_INFINITY, // The back-end does not accept transactions for mappings. -} as const; +export const MAXIMUM_ACTION_COUNT_PER_BATCH = 1000; // todop: should this be smarter? // export const MAXIMUM_ACTION_COUNT_PER_SAVE = { diff --git a/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx b/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx index 016a1feabcd..43e292c032d 100644 --- a/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx +++ b/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx @@ -947,11 +947,7 @@ function* handleDeleteSegmentData(): Saga { yield* put(setBusyBlockingInfoAction(true, "Segment is being deleted.")); yield* put( - pushSaveQueueTransaction( - [deleteSegmentDataVolumeAction(action.segmentId)], - "volume", - action.layerName, - ), + pushSaveQueueTransaction([deleteSegmentDataVolumeAction(action.segmentId)], action.layerName), ); yield* call([Model, Model.ensureSavedState]); diff --git a/frontend/javascripts/oxalis/model_initialization.ts b/frontend/javascripts/oxalis/model_initialization.ts index 5ff7619bfab..d8b8fbcd97d 100644 --- a/frontend/javascripts/oxalis/model_initialization.ts +++ b/frontend/javascripts/oxalis/model_initialization.ts @@ -11,7 +11,6 @@ import type { APICompoundType, APISegmentationLayer, } from "types/api_flow_types"; -import type { Versions } from "oxalis/view/version_view"; import { computeDataTexturesSetup, getSupportedTextureSpecs, @@ -106,6 +105,7 @@ import { isFeatureAllowedByPricingPlan, } from "admin/organization/pricing_plan_utils"; import { convertServerAdditionalAxesToFrontEnd } from "./model/reducers/reducer_helpers"; +import { setVersionNumberAction } from "./model/actions/save_actions"; export const HANDLED_ERROR = "error_was_handled"; type DataLayerCollection = Record; @@ -114,7 +114,7 @@ export async function initialize( initialMaybeCompoundType: APICompoundType | null, initialCommandType: TraceOrViewCommand, initialFetch: boolean, - versions?: Versions, + version?: number | undefined | null, ): Promise< | { dataLayers: DataLayerCollection; @@ -169,7 +169,7 @@ export async function initialize( const [dataset, initialUserSettings, serverTracings] = await fetchParallel( annotation, datasetId, - versions, + version, ); const serverVolumeTracings = getServerVolumeTracings(serverTracings); const serverVolumeTracingIds = serverVolumeTracings.map((volumeTracing) => volumeTracing.id); @@ -237,12 +237,12 @@ export async function initialize( async function fetchParallel( annotation: APIAnnotation | null | undefined, datasetId: APIDatasetId, - versions?: Versions, + version: number | undefined | null, ): Promise<[APIDataset, UserConfiguration, Array]> { return Promise.all([ getDataset(datasetId, getSharingTokenFromUrlParameters()), getUserConfiguration(), // Fetch the actual tracing from the datastore, if there is an skeletonAnnotation - annotation ? getTracingsForAnnotation(annotation, versions) : [], + annotation ? getTracingsForAnnotation(annotation, version) : [], ]); } @@ -294,6 +294,7 @@ function initializeTracing( // This method is not called for the View mode const { dataset } = Store.getState(); let annotation = _annotation; + let version = 0; const { allowedModes, preferredMode } = determineAllowedModes(annotation.settings); _.extend(annotation.settings, { @@ -325,6 +326,7 @@ function initializeTracing( getSegmentationLayers(dataset).length > 0, messages["tracing.volume_missing_segmentation"], ); + version = Math.max(version, volumeTracing.version); Store.dispatch(initializeVolumeTracingAction(volumeTracing)); }); @@ -336,8 +338,10 @@ function initializeTracing( // To generate a huge amount of dummy trees, use: // import generateDummyTrees from "./model/helpers/generate_dummy_trees"; // tracing.trees = generateDummyTrees(1, 200000); + version = Math.max(version, skeletonTracing.version); Store.dispatch(initializeSkeletonTracingAction(skeletonTracing)); } + Store.dispatch(setVersionNumberAction(version)); } // Initialize 'flight', 'oblique' or 'orthogonal' mode @@ -464,6 +468,7 @@ function initializeDataLayerInstances(gpuFactor: number | null | undefined): { layer, textureInformation.textureSize, textureInformation.textureCount, + layer.name, // In case of a volume tracing layer the layer name will equal its tracingId. ); } diff --git a/frontend/javascripts/oxalis/store.ts b/frontend/javascripts/oxalis/store.ts index 28c00f472b4..ea4e169f073 100644 --- a/frontend/javascripts/oxalis/store.ts +++ b/frontend/javascripts/oxalis/store.ts @@ -28,7 +28,7 @@ import type { AdditionalAxis, MetadataEntryProto, } from "types/api_flow_types"; -import type { TracingStats } from "oxalis/model/accessors/annotation_accessor"; +import type { CombinedTracingStats } from "oxalis/model/accessors/annotation_accessor"; import type { Action } from "oxalis/model/actions/actions"; import type { BoundingBoxType, @@ -50,7 +50,7 @@ import type { } from "oxalis/constants"; import type { BLEND_MODES, ControlModeEnum } from "oxalis/constants"; import type { Matrix4x4 } from "libs/mjs"; -import type { UpdateAction, UpdateActionWithTracingId } from "oxalis/model/sagas/update_actions"; +import type { UpdateActionWithTracingId } from "oxalis/model/sagas/update_actions"; import AnnotationReducer from "oxalis/model/reducers/annotation_reducer"; import DatasetReducer from "oxalis/model/reducers/dataset_reducer"; import type DiffableMap from "libs/diffable_map"; @@ -191,6 +191,7 @@ export type AnnotationVisibility = APIAnnotationVisibility; export type RestrictionsAndSettings = Restrictions & Settings; export type Annotation = { readonly annotationId: string; + readonly version: number; readonly restrictions: RestrictionsAndSettings; readonly visibility: AnnotationVisibility; readonly annotationLayers: Array; @@ -449,7 +450,7 @@ export type SaveQueueEntry = { transactionId: string; transactionGroupCount: number; transactionGroupIndex: number; - stats: TracingStats | null | undefined; + stats: CombinedTracingStats | null | undefined; info: string; }; export type ProgressInfo = { diff --git a/frontend/javascripts/oxalis/view/action-bar/save_button.tsx b/frontend/javascripts/oxalis/view/action-bar/save_button.tsx index ab12ab6fd61..d08db492659 100644 --- a/frontend/javascripts/oxalis/view/action-bar/save_button.tsx +++ b/frontend/javascripts/oxalis/view/action-bar/save_button.tsx @@ -13,7 +13,6 @@ import { LoadingOutlined, } from "@ant-design/icons"; import ErrorHandling from "libs/error_handling"; -import * as Utils from "libs/utils"; import FastTooltip from "components/fast_tooltip"; import { Tooltip } from "antd"; import { reuseInstanceOnEquality } from "oxalis/model/accessors/accessor_helpers"; diff --git a/frontend/javascripts/oxalis/view/right-border-tabs/dataset_info_tab_view.tsx b/frontend/javascripts/oxalis/view/right-border-tabs/dataset_info_tab_view.tsx index 5a9da5127fb..bebf6409f4e 100644 --- a/frontend/javascripts/oxalis/view/right-border-tabs/dataset_info_tab_view.tsx +++ b/frontend/javascripts/oxalis/view/right-border-tabs/dataset_info_tab_view.tsx @@ -13,10 +13,7 @@ import { getResolutionUnion, } from "oxalis/model/accessors/dataset_accessor"; import { getActiveResolutionInfo } from "oxalis/model/accessors/flycam_accessor"; -import { - getCombinedStats, - type CombinedTracingStats, -} from "oxalis/model/accessors/annotation_accessor"; +import { getStats, type CombinedTracingStats } from "oxalis/model/accessors/annotation_accessor"; import { setAnnotationNameAction, setAnnotationDescriptionAction, @@ -272,7 +269,7 @@ export class DatasetInfoTabView extends React.PureComponent { getAnnotationStatistics() { if (this.props.isDatasetViewMode) return null; - return ; + return ; } getKeyboardShortcuts() { diff --git a/frontend/javascripts/oxalis/view/right-border-tabs/skeleton_tab_view.tsx b/frontend/javascripts/oxalis/view/right-border-tabs/skeleton_tab_view.tsx index 7a2f1b7099e..99e672661cf 100644 --- a/frontend/javascripts/oxalis/view/right-border-tabs/skeleton_tab_view.tsx +++ b/frontend/javascripts/oxalis/view/right-border-tabs/skeleton_tab_view.tsx @@ -244,13 +244,7 @@ export async function importTracingFiles(files: Array, createGroupForEachF if (oldVolumeTracing) { Store.dispatch(importVolumeTracingAction()); - Store.dispatch( - setVersionNumberAction( - oldVolumeTracing.version + 1, - "volume", - oldVolumeTracing.tracingId, - ), - ); + Store.dispatch(setVersionNumberAction(tracing.version + 1)); Store.dispatch(setLargestSegmentIdAction(newLargestSegmentId)); await clearCache(dataset, oldVolumeTracing.tracingId); await api.data.reloadBuckets(oldVolumeTracing.tracingId); diff --git a/frontend/javascripts/oxalis/view/version_entry.tsx b/frontend/javascripts/oxalis/view/version_entry.tsx index f9133b26a2f..e363c9060b1 100644 --- a/frontend/javascripts/oxalis/view/version_entry.tsx +++ b/frontend/javascripts/oxalis/view/version_entry.tsx @@ -38,12 +38,14 @@ import type { MergeTreeUpdateAction, UpdateMappingNameUpdateAction, DeleteSegmentDataUpdateAction, + UpdateActionWithTracingId, } from "oxalis/model/sagas/update_actions"; import FormattedDate from "components/formatted_date"; import { MISSING_GROUP_ID } from "oxalis/view/right-border-tabs/tree_hierarchy_view_helpers"; import { useSelector } from "react-redux"; -import type { OxalisState } from "oxalis/store"; +import type { HybridTracing, OxalisState } from "oxalis/store"; import { formatUserName, getContributorById } from "oxalis/model/accessors/user_accessor"; +import { getReadableNameByVolumeTracingId } from "oxalis/model/accessors/volumetracing_accessor"; type Description = { description: string; icon: React.ReactNode; @@ -56,7 +58,10 @@ const updateTracingDescription = { // determines the order in which update actions are checked // to describe an update action batch. See also the comment // of the `getDescriptionForBatch` function. -const descriptionFns: Record Description> = { +const descriptionFns: Record< + ServerUpdateAction["name"], + (firstAction: any, actionCount: number, tracing: HybridTracing) => Description +> = { importVolumeTracing: (): Description => ({ description: "Imported a volume tracing.", icon: , @@ -122,14 +127,28 @@ const descriptionFns: Record Descr description: `Updated the tree with id ${action.value.id}.`, icon: , }), - updateBucket: (): Description => ({ - description: "Updated the segmentation.", - icon: , - }), - updateSegmentGroups: (): Description => ({ - description: "Updated the segment groups.", - icon: , - }), + updateBucket: ( + firstAction: UpdateActionWithTracingId, + _actionCount: number, + tracing: HybridTracing, + ): Description => { + const layerName = maybeGetReadableVolumeTracingName(tracing, firstAction.value.actionTracingId); + return { + description: `Updated the segmentation of layer ${layerName}.`, + icon: , + }; + }, + updateSegmentGroups: ( + firstAction: UpdateActionWithTracingId, + _actionCount: number, + tracing: HybridTracing, + ): Description => { + const layerName = maybeGetReadableVolumeTracingName(tracing, firstAction.value.actionTracingId); + return { + description: `Updated the segment groups of layer ${layerName}.`, + icon: , + }; + }, updateNode: (action: UpdateNodeUpdateAction): Description => ({ description: `Updated the node with id ${action.value.id}.`, icon: , @@ -160,26 +179,61 @@ const descriptionFns: Record Descr description: "Updated the 3D view.", icon: , }), - createSegment: (action: CreateSegmentUpdateAction): Description => ({ - description: `Added the segment with id ${action.value.id} to the segments list.`, - icon: , - }), - updateSegment: (action: UpdateSegmentUpdateAction): Description => ({ - description: `Updated the segment with id ${action.value.id} in the segments list.`, - icon: , - }), - deleteSegment: (action: DeleteSegmentUpdateAction): Description => ({ - description: `Deleted the segment with id ${action.value.id} from the segments list.`, - icon: , - }), - deleteSegmentData: (action: DeleteSegmentDataUpdateAction): Description => ({ - description: `Deleted the data of segment ${action.value.id}. All voxels with that id were overwritten with 0.`, - icon: , - }), - addSegmentIndex: (): Description => ({ - description: "Added segment index to enable segment statistics.", - icon: , - }), + createSegment: ( + firstAction: UpdateActionWithTracingId & CreateSegmentUpdateAction, + _actionCount: number, + tracing: HybridTracing, + ): Description => { + const layerName = maybeGetReadableVolumeTracingName(tracing, firstAction.value.actionTracingId); + return { + description: `Added the segment with id ${firstAction.value.id} to the segments list of layer ${layerName}.`, + icon: , + }; + }, + updateSegment: ( + firstAction: UpdateActionWithTracingId & UpdateSegmentUpdateAction, + _actionCount: number, + tracing: HybridTracing, + ): Description => { + const layerName = maybeGetReadableVolumeTracingName(tracing, firstAction.value.actionTracingId); + return { + description: `Updated the segment with id ${firstAction.value.id} in the segments list of layer ${layerName}.`, + icon: , + }; + }, + deleteSegment: ( + firstAction: UpdateActionWithTracingId & DeleteSegmentUpdateAction, + _actionCount: number, + tracing: HybridTracing, + ): Description => { + const layerName = maybeGetReadableVolumeTracingName(tracing, firstAction.value.actionTracingId); + return { + description: `Deleted the segment with id ${firstAction.value.id} from the segments list of layer ${layerName}.`, + icon: , + }; + }, + deleteSegmentData: ( + firstAction: UpdateActionWithTracingId & DeleteSegmentDataUpdateAction, + _actionCount: number, + tracing: HybridTracing, + ): Description => { + const layerName = maybeGetReadableVolumeTracingName(tracing, firstAction.value.actionTracingId); + return { + description: `Deleted the data of segment ${firstAction.value.id} of layer ${layerName}. All voxels with that id were overwritten with 0.`, + icon: , + }; + }, + addSegmentIndex: ( + firstAction: UpdateActionWithTracingId, + _actionCount: number, + tracing: HybridTracing, + ): Description => { + const layerName = maybeGetReadableVolumeTracingName(tracing, firstAction.value.actionTracingId); + return { + description: `Added segment index to layer ${layerName} to enable segment statistics.`, + icon: , + }; + }, // This should never be shown since currently this update action can only be triggered // by merging or splitting trees which is recognized separately, before this description // is accessed. @@ -196,9 +250,17 @@ const descriptionFns: Record Descr updateVolumeTracing: (): Description => updateTracingDescription, } as const; +function maybeGetReadableVolumeTracingName(tracing: HybridTracing, tracingId: string): string { + const volumeTracing = tracing.volumes.find((volume) => volume.tracingId === tracingId); + return volumeTracing != null + ? getReadableNameByVolumeTracingId(tracing, volumeTracing.tracingId) + : ""; +} + function getDescriptionForSpecificBatch( actions: Array, type: string, + tracing: HybridTracing, ): Description { const firstAction = actions[0]; @@ -206,7 +268,7 @@ function getDescriptionForSpecificBatch( throw new Error("Type constraint violated"); } const fn = descriptionFns[type]; - return fn(firstAction, actions.length); + return fn(firstAction, actions.length, tracing); } // An update action batch can consist of more than one update action as a single user action @@ -220,7 +282,10 @@ function getDescriptionForSpecificBatch( // "more expressive" update actions first and for more general ones later. // The order is determined by the order in which the update actions are added to the // `descriptionFns` object. -function getDescriptionForBatch(actions: Array): Description { +function getDescriptionForBatch( + actions: Array, + tracing: HybridTracing, +): Description { const groupedUpdateActions = _.groupBy(actions, "name"); const moveTreeComponentUAs = groupedUpdateActions.moveTreeComponent; @@ -270,7 +335,7 @@ function getDescriptionForBatch(actions: Array): Description const updateActions = groupedUpdateActions[key]; if (updateActions != null) { - return getDescriptionForSpecificBatch(updateActions, key); + return getDescriptionForSpecificBatch(updateActions, key, tracing); } } @@ -302,6 +367,7 @@ export default function VersionEntry({ const contributors = useSelector((state: OxalisState) => state.tracing.contributors); const activeUser = useSelector((state: OxalisState) => state.activeUser); const owner = useSelector((state: OxalisState) => state.tracing.owner); + const tracing = useSelector((state: OxalisState) => state.tracing); const liClassName = classNames("version-entry", { "active-version-entry": isActive, @@ -317,7 +383,7 @@ export default function VersionEntry({ {allowUpdate ? "Restore" : "Download"} ); - const { description, icon } = getDescriptionForBatch(actions); + const { description, icon } = getDescriptionForBatch(actions, tracing); // In case the actionAuthorId is not set, the action was created before the multi-contributor // support. Default to the owner in that case. diff --git a/frontend/javascripts/oxalis/view/version_list.tsx b/frontend/javascripts/oxalis/view/version_list.tsx index c0d8a274830..20928c8345a 100644 --- a/frontend/javascripts/oxalis/view/version_list.tsx +++ b/frontend/javascripts/oxalis/view/version_list.tsx @@ -3,7 +3,6 @@ import { useState, useEffect } from "react"; import _ from "lodash"; import dayjs from "dayjs"; import type { APIUpdateActionBatch } from "types/api_flow_types"; -import type { Versions } from "oxalis/view/version_view"; import { chunkIntoTimeWindows } from "libs/utils"; import { getUpdateActionLog, @@ -13,7 +12,6 @@ import { import { handleGenericError } from "libs/error_handling"; import { pushSaveQueueTransaction, - type SaveQueueType, setVersionNumberAction, } from "oxalis/model/actions/save_actions"; import { @@ -24,21 +22,19 @@ import { import { setAnnotationAllowUpdateAction } from "oxalis/model/actions/annotation_actions"; import { setVersionRestoreVisibilityAction } from "oxalis/model/actions/ui_actions"; import { Model } from "oxalis/singletons"; -import type { EditableMapping, OxalisState, SkeletonTracing, VolumeTracing } from "oxalis/store"; +import type { HybridTracing, OxalisState } from "oxalis/store"; import Store from "oxalis/store"; import VersionEntryGroup from "oxalis/view/version_entry_group"; import { api } from "oxalis/singletons"; -import Toast from "libs/toast"; import { useInfiniteQuery, useQueryClient } from "@tanstack/react-query"; import { useEffectOnlyOnce } from "libs/react_hooks"; import { useFetch } from "libs/react_helpers"; import { useSelector } from "react-redux"; +import { getCreationTimestamp } from "oxalis/model/accessors/annotation_accessor"; const ENTRIES_PER_PAGE = 5000; type Props = { - versionedObjectType: SaveQueueType; - tracing: SkeletonTracing | VolumeTracing | EditableMapping; allowUpdate: boolean; }; @@ -49,25 +45,18 @@ type GroupedAndChunkedVersions = Record - Model.getSegmentationTracingLayer(volumeTracingId), - ); - segmentationLayersToReload.push(...versionedSegmentationLayers); - } + // TODO: properly determine which layers to reload. + // No versions were passed which means that the newest annotation should be + // shown. Therefore, reload all segmentation layers. + segmentationLayersToReload.push(...Model.getSegmentationTracingLayers()); for (const segmentationLayer of segmentationLayersToReload) { segmentationLayer.cube.collectAllBuckets(); @@ -80,20 +69,13 @@ async function handleRestoreVersion( versions: APIUpdateActionBatch[], version: number, ) { - const getNewestVersion = () => _.max(versions.map((batch) => batch.version)) || 0; if (props.allowUpdate) { - Store.dispatch( - setVersionNumberAction( - getNewestVersion(), - props.versionedObjectType, - props.tracing.tracingId, - ), - ); + const newestVersion = _.max(versions.map((batch) => batch.version)) || 0; + Store.dispatch(setVersionNumberAction(newestVersion)); Store.dispatch( pushSaveQueueTransaction( [revertToVersion(version)], - props.versionedObjectType, - props.tracing.tracingId, + "experimental; leaving out tracingId as this should not be required", ), ); await Model.ensureSavedState(); @@ -102,28 +84,7 @@ async function handleRestoreVersion( } else { const { annotationType, annotationId, volumes } = Store.getState().tracing; const includesVolumeFallbackData = volumes.some((volume) => volume.fallbackLayer != null); - downloadAnnotation(annotationId, annotationType, includesVolumeFallbackData, { - [props.versionedObjectType]: version, - }); - } -} - -function handlePreviewVersion(props: Props, version: number) { - if (props.versionedObjectType === "skeleton") { - return previewVersion({ - skeleton: version, - }); - } else if (props.versionedObjectType === "volume") { - return previewVersion({ - volumes: { - [props.tracing.tracingId]: version, - }, - }); - } else { - Toast.warning( - `Version preview and restoring for ${props.versionedObjectType}s is not supported yet.`, - ); - return Promise.resolve(); + downloadAnnotation(annotationId, annotationType, includesVolumeFallbackData, version); } } @@ -146,7 +107,7 @@ const getGroupedAndChunkedVersions = _.memoize( ); async function getUpdateActionLogPage( - props: Props, + tracing: HybridTracing, tracingStoreUrl: string, annotationId: string, newestVersion: number, @@ -186,7 +147,7 @@ async function getUpdateActionLogPage( if (oldestVersionInPage === 1) { updateActionLog.push({ version: 0, - value: [serverCreateTracing(props.tracing.createdTimestamp)], + value: [serverCreateTracing(getCreationTimestamp(tracing))], }); } @@ -199,15 +160,15 @@ async function getUpdateActionLogPage( } function VersionList(props: Props) { - const { tracing } = props; const tracingStoreUrl = useSelector((state: OxalisState) => state.tracing.tracingStore.url); const annotationId = useSelector((state: OxalisState) => state.tracing.annotationId); const newestVersion = useFetch( () => getNewestVersionForTracing(tracingStoreUrl, annotationId), null, - [tracing], + [annotationId], ); + console.log("newestVersion", newestVersion); if (newestVersion == null) { return ( @@ -221,24 +182,26 @@ function VersionList(props: Props) { } function InnerVersionList(props: Props & { newestVersion: number }) { + const tracing = useSelector((state: OxalisState) => state.tracing); const queryClient = useQueryClient(); // Remember the version with which the version view was opened ( // the active version could change by the actions of the user). // Based on this version, the page numbers are calculated. const { newestVersion } = props; - const [initialVersion] = useState(props.tracing.version); + const [initialVersion] = useState(tracing.version); function fetchPaginatedVersions({ pageParam }: { pageParam?: number }) { + // TODO: maybe refactor this so that this method is not calculated very rendering cycle if (pageParam == null) { pageParam = Math.floor((newestVersion - initialVersion) / ENTRIES_PER_PAGE); } const { url: tracingStoreUrl } = Store.getState().tracing.tracingStore; const annotationId = Store.getState().tracing.annotationId; - return getUpdateActionLogPage(props, tracingStoreUrl, annotationId, newestVersion, pageParam); + return getUpdateActionLogPage(tracing, tracingStoreUrl, annotationId, newestVersion, pageParam); } - const queryKey = ["versions", props.tracing.tracingId]; + const queryKey = ["versions", tracing.annotationId]; useEffectOnlyOnce(() => { // Remove all previous existent queries so that the content of this view @@ -330,11 +293,11 @@ function InnerVersionList(props: Props & { newestVersion: number }) { batches={batchesOrDateString} allowUpdate={props.allowUpdate} newestVersion={flattenedVersions[0].version} - activeVersion={props.tracing.version} + activeVersion={tracing.version} onRestoreVersion={(version) => handleRestoreVersion(props, flattenedVersions, version) } - onPreviewVersion={(version) => handlePreviewVersion(props, version)} + onPreviewVersion={(version) => previewVersion(version)} key={batchesOrDateString[0].version} /> ) diff --git a/frontend/javascripts/oxalis/view/version_view.tsx b/frontend/javascripts/oxalis/view/version_view.tsx index 4a8862b7719..e755a6c6f38 100644 --- a/frontend/javascripts/oxalis/view/version_view.tsx +++ b/frontend/javascripts/oxalis/view/version_view.tsx @@ -1,14 +1,14 @@ -import { Button, Alert, Tabs, type TabsProps } from "antd"; +import { Button, Alert } from "antd"; import { CloseOutlined } from "@ant-design/icons"; -import { connect } from "react-redux"; +import { connect, useDispatch } from "react-redux"; import * as React from "react"; -import { getReadableNameByVolumeTracingId } from "oxalis/model/accessors/volumetracing_accessor"; import { setAnnotationAllowUpdateAction } from "oxalis/model/actions/annotation_actions"; import { setVersionRestoreVisibilityAction } from "oxalis/model/actions/ui_actions"; import type { OxalisState, Tracing } from "oxalis/store"; -import { type TracingType, TracingTypeEnum } from "types/api_flow_types"; import Store from "oxalis/store"; import VersionList, { previewVersion } from "oxalis/view/version_list"; +import { useState } from "react"; +import { useWillUnmount } from "beautiful-react-hooks"; export type Versions = { skeleton?: number | null | undefined; @@ -21,151 +21,84 @@ type OwnProps = { allowUpdate: boolean; }; type Props = StateProps & OwnProps; -type State = { - activeTracingType: TracingType; - initialAllowUpdate: boolean; -}; -class VersionView extends React.Component { - state: State = { - activeTracingType: - this.props.tracing.skeleton != null ? TracingTypeEnum.skeleton : TracingTypeEnum.volume, - // Remember whether the tracing could originally be updated - initialAllowUpdate: this.props.allowUpdate, - }; +const VersionView: React.FC = (props: Props) => { + const [initialAllowUpdate] = useState(props.allowUpdate); + const dispatch = useDispatch(); - componentWillUnmount() { - Store.dispatch(setAnnotationAllowUpdateAction(this.state.initialAllowUpdate)); - } + useWillUnmount(() => { + dispatch(setAnnotationAllowUpdateAction(initialAllowUpdate)); + }); - handleClose = async () => { + const handleClose = async () => { // This will load the newest version of both skeleton and volume tracings await previewVersion(); Store.dispatch(setVersionRestoreVisibilityAction(false)); - Store.dispatch(setAnnotationAllowUpdateAction(this.state.initialAllowUpdate)); - }; - - onChangeTab = (activeKey: string) => { - this.setState({ - activeTracingType: activeKey as TracingType, - }); + Store.dispatch(setAnnotationAllowUpdateAction(initialAllowUpdate)); }; - render() { - const tabs: TabsProps["items"] = []; - - if (this.props.tracing.skeleton != null) - tabs.push({ - label: "Skeleton", - key: "skeleton", - children: ( - - ), - }); - - tabs.push( - ...this.props.tracing.volumes.map((volumeTracing) => ({ - label: getReadableNameByVolumeTracingId(this.props.tracing, volumeTracing.tracingId), - key: volumeTracing.tracingId, - children: ( - - ), - })), - ); - - tabs.push( - ...this.props.tracing.mappings.map((mapping) => ({ - label: `${getReadableNameByVolumeTracingId( - this.props.tracing, - mapping.tracingId, - )} (Editable Mapping)`, - key: mapping.tracingId, - children: ( - - ), - })), - ); - - return ( + return ( +
-
-

- Version History -

-
+ Version History + +
- ); - } -} +
+ +
+
+ ); +}; function mapStateToProps(state: OxalisState): StateProps { return { diff --git a/frontend/javascripts/types/api_flow_types.ts b/frontend/javascripts/types/api_flow_types.ts index de37bc59b18..0ba4b896605 100644 --- a/frontend/javascripts/types/api_flow_types.ts +++ b/frontend/javascripts/types/api_flow_types.ts @@ -551,6 +551,7 @@ type APIAnnotationBase = APIAnnotationInfo & { readonly owner?: APIUserBase; // This `user` attribute is deprecated and should not be used, anymore. It only exists to satisfy e2e type checks readonly user?: APIUserBase; + readonly version: number; readonly contributors: APIUserBase[]; readonly othersMayEdit: boolean; }; diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayerType.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayerType.scala index 2f34d15b159..9b1a7dd2d9d 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayerType.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayerType.scala @@ -17,5 +17,7 @@ object AnnotationLayerType extends ExtendedEnumeration { p match { case AnnotationLayerTypeProto.skeleton => Skeleton case AnnotationLayerTypeProto.volume => Volume + case AnnotationLayerTypeProto.Unrecognized(_) => + Volume // unrecognized should never happen, artifact of proto code generation } } From 90d8e4d8fa298be99cf25e7819f2193f48fba62b Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 21 Oct 2024 16:40:30 +0200 Subject: [PATCH 112/150] AnnotationDefaults --- app/models/annotation/Annotation.scala | 9 +++++++-- app/models/annotation/AnnotationService.scala | 6 ++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/app/models/annotation/Annotation.scala b/app/models/annotation/Annotation.scala index 493bce4bdd4..d608250a649 100755 --- a/app/models/annotation/Annotation.scala +++ b/app/models/annotation/Annotation.scala @@ -22,6 +22,11 @@ import javax.inject.Inject import scala.concurrent.ExecutionContext import scala.concurrent.duration.FiniteDuration +object AnnotationDefaults { + val defaultName: String = "" + val defaultDescription: String = "" +} + case class Annotation( _id: ObjectId, _dataset: ObjectId, @@ -29,9 +34,9 @@ case class Annotation( _team: ObjectId, _user: ObjectId, annotationLayers: List[AnnotationLayer], - description: String = "", + description: String = AnnotationDefaults.defaultDescription, visibility: AnnotationVisibility.Value = AnnotationVisibility.Internal, - name: String = "", + name: String = AnnotationDefaults.defaultName, viewConfiguration: Option[JsObject] = None, state: AnnotationState.Value = Active, isLockedByOwner: Boolean = false, diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index bda33814f37..2c7e94a3397 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -278,8 +278,10 @@ class AnnotationService @Inject()( AnnotationLayerType.toProto(l.typ) ) } - //TODO pass right name, description here - annotationProto = AnnotationProto(name = None, description = None, version = 0L, layers = layersProto) + annotationProto = AnnotationProto(name = Some(AnnotationDefaults.defaultName), + description = Some(AnnotationDefaults.defaultDescription), + version = 0L, + layers = layersProto) _ <- tracingStoreClient.saveAnnotationProto(annotationId, annotationProto) } yield newAnnotationLayers From 759c2f30b223cbc32c4a476bb97c133ddb44e258 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 23 Oct 2024 09:49:35 +0200 Subject: [PATCH 113/150] fix after merge. report name + description to wk --- .../WKRemoteTracingStoreController.scala | 22 +++++-- conf/webknossos.latest.routes | 2 +- .../oxalis/model/sagas/save_saga.ts | 60 ++----------------- .../dataset_info_tab_view.tsx | 16 +---- .../com/scalableminds/util/tools/Fox.scala | 2 +- .../models/annotation/AnnotationLayer.scala | 1 + .../TSRemoteWebknossosClient.scala | 7 +++ .../annotation/TSAnnotationService.scala | 6 +- 8 files changed, 32 insertions(+), 84 deletions(-) diff --git a/app/controllers/WKRemoteTracingStoreController.scala b/app/controllers/WKRemoteTracingStoreController.scala index eebbf594f88..1e05a4df3f5 100644 --- a/app/controllers/WKRemoteTracingStoreController.scala +++ b/app/controllers/WKRemoteTracingStoreController.scala @@ -3,6 +3,7 @@ package controllers import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext} import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.{Fox, FoxImplicits} +import com.scalableminds.webknossos.datastore.Annotation.AnnotationProto import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayer @@ -17,6 +18,7 @@ import models.annotation.AnnotationState._ import models.annotation.{ Annotation, AnnotationDAO, + AnnotationDefaults, AnnotationInformationProvider, AnnotationLayerDAO, AnnotationService, @@ -58,23 +60,31 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore val bearerTokenService: WebknossosBearerTokenAuthenticatorService = wkSilhouetteEnvironment.combinedAuthenticatorService.tokenAuthenticatorService - def updateAnnotationLayers(name: String, key: String, annotationId: String): Action[List[AnnotationLayer]] = - Action.async(validateJson[List[AnnotationLayer]]) { implicit request => + def updateAnnotation(name: String, key: String, annotationId: String): Action[AnnotationProto] = + Action.async(validateProto[AnnotationProto]) { implicit request => + // tracingstore only sends this request after ensuring write access + implicit val ctx: DBAccessContext = GlobalAccessContext for { annotationIdValidated <- ObjectId.fromString(annotationId) existingLayers <- annotationLayerDAO.findAnnotationLayersFor(annotationIdValidated) + newLayersProto = request.body.layers existingLayerIds = existingLayers.map(_.tracingId).toSet - newLayerIds = request.body.map(_.tracingId).toSet + newLayerIds = newLayersProto.map(_.tracingId).toSet layerIdsToDelete = existingLayerIds.diff(newLayerIds) layerIdsToUpdate = existingLayerIds.intersect(newLayerIds) layerIdsToInsert = newLayerIds.diff(existingLayerIds) _ <- Fox.serialCombined(layerIdsToDelete.toList)( annotationLayerDAO.deleteOneByTracingId(annotationIdValidated, _)) - _ <- Fox.serialCombined(request.body.filter(l => layerIdsToInsert.contains(l.tracingId)))( - annotationLayerDAO.insertOne(annotationIdValidated, _)) - _ <- Fox.serialCombined(request.body.filter(l => layerIdsToUpdate.contains(l.tracingId)))(l => + _ <- Fox.serialCombined(newLayersProto.filter(l => layerIdsToInsert.contains(l.tracingId))) { layerProto => + annotationLayerDAO.insertOne(annotationIdValidated, AnnotationLayer.fromProto(layerProto)) + } + _ <- Fox.serialCombined(newLayersProto.filter(l => layerIdsToUpdate.contains(l.tracingId)))(l => annotationLayerDAO.updateName(annotationIdValidated, l.tracingId, l.name)) // Layer stats are ignored here, they are sent eagerly when saving updates + _ <- annotationDAO.updateName(annotationIdValidated, + request.body.name.getOrElse(AnnotationDefaults.defaultName)) + _ <- annotationDAO.updateDescription(annotationIdValidated, + request.body.description.getOrElse(AnnotationDefaults.defaultDescription)) } yield Ok } diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes index a1ace976868..e439b682927 100644 --- a/conf/webknossos.latest.routes +++ b/conf/webknossos.latest.routes @@ -121,7 +121,7 @@ PUT /datastores/:name # Tracingstores GET /tracingstore controllers.TracingStoreController.listOne() POST /tracingstores/:name/handleTracingUpdateReport controllers.WKRemoteTracingStoreController.handleTracingUpdateReport(name: String, key: String) -POST /tracingstores/:name/updateAnnotationLayers controllers.WKRemoteTracingStoreController.updateAnnotationLayers(name: String, key: String, annotationId: String) +POST /tracingstores/:name/updateAnnotation controllers.WKRemoteTracingStoreController.updateAnnotation(name: String, key: String, annotationId: String) POST /tracingstores/:name/validateUserAccess controllers.UserTokenController.validateAccessViaTracingstore(name: String, key: String, token: Option[String]) PUT /tracingstores/:name controllers.TracingStoreController.update(name: String) GET /tracingstores/:name/dataSource controllers.WKRemoteTracingStoreController.dataSourceForTracing(name: String, key: String, tracingId: String) diff --git a/frontend/javascripts/oxalis/model/sagas/save_saga.ts b/frontend/javascripts/oxalis/model/sagas/save_saga.ts index 68ad8eabd3d..d7f7411738e 100644 --- a/frontend/javascripts/oxalis/model/sagas/save_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/save_saga.ts @@ -9,17 +9,8 @@ import window, { alert, document, location } from "libs/window"; import _ from "lodash"; import messages from "messages"; import { ControlModeEnum } from "oxalis/constants"; -<<<<<<< HEAD -import { getResolutionInfo } from "oxalis/model/accessors/dataset_accessor"; -||||||| 934bb6aa9b -import { getResolutionInfo } from "oxalis/model/accessors/dataset_accessor"; -import { selectQueue } from "oxalis/model/accessors/save_accessor"; -======= import { getMagInfo } from "oxalis/model/accessors/dataset_accessor"; -import { selectQueue } from "oxalis/model/accessors/save_accessor"; ->>>>>>> master import { selectTracing } from "oxalis/model/accessors/tracing_accessor"; -import { getVolumeTracingById } from "oxalis/model/accessors/volumetracing_accessor"; import { FlycamActions } from "oxalis/model/actions/flycam_actions"; import { pushSaveQueueTransaction, @@ -286,7 +277,6 @@ export function* sendSaveRequestToServer(): Saga { } } -<<<<<<< HEAD function* markBucketsAsNotDirty(saveQueue: Array) { for (const saveEntry of saveQueue) { for (const updateAction of saveEntry.actions) { @@ -295,59 +285,17 @@ function* markBucketsAsNotDirty(saveQueue: Array) { // an updateBucket action. const { actionTracingId: tracingId } = updateAction.value; const segmentationLayer = Model.getSegmentationTracingLayer(tracingId); - const segmentationResolutionInfo = yield* call( - getResolutionInfo, - segmentationLayer.resolutions, - ); -||||||| 934bb6aa9b -function* markBucketsAsNotDirty(saveQueue: Array, tracingId: string) { - const segmentationLayer = Model.getSegmentationTracingLayer(tracingId); - const segmentationResolutionInfo = yield* call(getResolutionInfo, segmentationLayer.resolutions); -======= -function* markBucketsAsNotDirty(saveQueue: Array, tracingId: string) { - const segmentationLayer = Model.getSegmentationTracingLayer(tracingId); - const segmentationResolutionInfo = yield* call(getMagInfo, segmentationLayer.resolutions); ->>>>>>> master - -<<<<<<< HEAD + const segmentationResolutionInfo = yield* call(getMagInfo, segmentationLayer.resolutions); + const { position, mag, additionalCoordinates } = updateAction.value; - const resolutionIndex = segmentationResolutionInfo.getIndexByResolution(mag); + const resolutionIndex = segmentationResolutionInfo.getIndexByMag(mag); const zoomedBucketAddress = globalPositionToBucketPosition( position, - segmentationResolutionInfo.getDenseResolutions(), + segmentationResolutionInfo.getDenseMags(), resolutionIndex, additionalCoordinates, ); const bucket = segmentationLayer.cube.getOrCreateBucket(zoomedBucketAddress); -||||||| 934bb6aa9b - if (segmentationLayer != null) { - for (const saveEntry of saveQueue) { - for (const updateAction of saveEntry.actions) { - if (updateAction.name === "updateBucket") { - const { position, mag, additionalCoordinates } = updateAction.value; - const resolutionIndex = segmentationResolutionInfo.getIndexByResolution(mag); - const zoomedBucketAddress = globalPositionToBucketPosition( - position, - segmentationResolutionInfo.getDenseResolutions(), - resolutionIndex, - additionalCoordinates, - ); - const bucket = segmentationLayer.cube.getOrCreateBucket(zoomedBucketAddress); -======= - if (segmentationLayer != null) { - for (const saveEntry of saveQueue) { - for (const updateAction of saveEntry.actions) { - if (updateAction.name === "updateBucket") { - const { position, mag, additionalCoordinates } = updateAction.value; - const resolutionIndex = segmentationResolutionInfo.getIndexByMag(mag); - const zoomedBucketAddress = globalPositionToBucketPosition( - position, - segmentationResolutionInfo.getDenseMags(), - resolutionIndex, - additionalCoordinates, - ); - const bucket = segmentationLayer.cube.getOrCreateBucket(zoomedBucketAddress); ->>>>>>> master if (bucket.type === "null") { continue; diff --git a/frontend/javascripts/oxalis/view/right-border-tabs/dataset_info_tab_view.tsx b/frontend/javascripts/oxalis/view/right-border-tabs/dataset_info_tab_view.tsx index 670bcf59df4..bf5b4f783cd 100644 --- a/frontend/javascripts/oxalis/view/right-border-tabs/dataset_info_tab_view.tsx +++ b/frontend/javascripts/oxalis/view/right-border-tabs/dataset_info_tab_view.tsx @@ -12,22 +12,8 @@ import { getDatasetExtentAsString, getMagnificationUnion, } from "oxalis/model/accessors/dataset_accessor"; -<<<<<<< HEAD -import { getActiveResolutionInfo } from "oxalis/model/accessors/flycam_accessor"; -import { getStats, type CombinedTracingStats } from "oxalis/model/accessors/annotation_accessor"; -||||||| 934bb6aa9b -import { getActiveResolutionInfo } from "oxalis/model/accessors/flycam_accessor"; -import { - getCombinedStats, - type CombinedTracingStats, -} from "oxalis/model/accessors/annotation_accessor"; -======= import { getActiveMagInfo } from "oxalis/model/accessors/flycam_accessor"; -import { - getCombinedStats, - type CombinedTracingStats, -} from "oxalis/model/accessors/annotation_accessor"; ->>>>>>> master +import { getStats, type CombinedTracingStats } from "oxalis/model/accessors/annotation_accessor"; import { setAnnotationNameAction, setAnnotationDescriptionAction, diff --git a/util/src/main/scala/com/scalableminds/util/tools/Fox.scala b/util/src/main/scala/com/scalableminds/util/tools/Fox.scala index 59af5b50d28..2195b751eef 100644 --- a/util/src/main/scala/com/scalableminds/util/tools/Fox.scala +++ b/util/src/main/scala/com/scalableminds/util/tools/Fox.scala @@ -133,7 +133,7 @@ object Fox extends FoxImplicits { } // Run serially, fail on the first failure - def serialCombined[A, B](l: List[A])(f: A => Fox[B])(implicit ec: ExecutionContext): Fox[List[B]] = + def serialCombined[A, B](l: Iterable[A])(f: A => Fox[B])(implicit ec: ExecutionContext): Fox[List[B]] = serialCombined(l.iterator)(f) // Run serially, fail on the first failure diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayer.scala index 1c8e827d31c..5b1ba9b6607 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayer.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayer.scala @@ -11,6 +11,7 @@ import scalapb.GeneratedMessage import scala.concurrent.ExecutionContext +// TODO can this be moved back to wk-core backend? case class AnnotationLayer( tracingId: String, typ: AnnotationLayerType, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala index 52ddf6f03f4..523daee164b 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala @@ -5,6 +5,7 @@ import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox +import com.scalableminds.webknossos.datastore.Annotation.AnnotationProto import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.models.annotation.{AnnotationLayer, AnnotationLayerType} @@ -99,6 +100,12 @@ class TSRemoteWebknossosClient @Inject()( .addQueryString("key" -> tracingStoreKey) .postJson(annotationLayers) + def updateAnnotation(annotationId: String, annotationProto: AnnotationProto): Fox[Unit] = + rpc(s"$webknossosUri/api/tracingstores/$tracingStoreName/updateAnnotation") + .addQueryString("annotationId" -> annotationId) + .addQueryString("key" -> tracingStoreKey) + .postProto(annotationProto) + def createTracingFor(annotationId: String, layerParameters: AnnotationLayerParameters, previousVersion: Long): Fox[Either[SkeletonTracing, VolumeTracing]] = { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 06386265681..c98ba090cb3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -10,7 +10,6 @@ import com.scalableminds.webknossos.datastore.EditableMappingInfo.EditableMappin import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits -import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayer import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ EditableMappingLayer, EditableMappingService, @@ -441,10 +440,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss _ <- updatedWithNewVerson.flushBufferedUpdates() _ <- flushUpdatedTracings(updatedWithNewVerson) _ <- flushAnnotationInfo(annotationId, updatedWithNewVerson) - _ <- remoteWebknossosClient.updateAnnotationLayers(annotationId, - updatedWithNewVerson.annotation.layers - .map(AnnotationLayer.fromProto) - .toList) // TODO perf: skip if no layer changes + _ <- remoteWebknossosClient.updateAnnotation(annotationId, updatedWithNewVerson.annotation) // TODO perf: skip if annotation is identical } yield updatedWithNewVerson } } From 1b8a3d51ac5b93566bc66e5956c5ef2719f6c5b3 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 23 Oct 2024 09:55:31 +0200 Subject: [PATCH 114/150] skip reporting changes to postgres when loading old versions --- .../annotation/TSAnnotationService.scala | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index c98ba090cb3..70a285483ce 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -204,6 +204,8 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss requestAll: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = for { targetVersion <- determineTargetVersion(annotationId, version) ?~> "determineTargetVersion.failed" + // When requesting any other than the newest version, do not consider the changes final + reportChangesToWk = version.isEmpty || version.contains(targetVersion) updatedAnnotation <- materializedAnnotationWithTracingCache.getOrLoad( (annotationId, targetVersion, requestedSkeletonTracingIds, requestedVolumeTracingIds, requestAll), _ => @@ -212,7 +214,8 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss targetVersion, requestedSkeletonTracingIds, requestedVolumeTracingIds, - requestAll = true) // TODO can we request fewer to save perf? still need to avoid duplicate apply + requestAll = true, + reportChangesToWk = reportChangesToWk) // TODO can we request fewer to save perf? still need to avoid duplicate apply ) } yield updatedAnnotation @@ -221,7 +224,8 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss version: Long, requestedSkeletonTracingIds: List[String], requestedVolumeTracingIds: List[String], - requestAll: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = + requestAll: Boolean, + reportChangesToWk: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = for { annotationWithVersion <- tracingDataStore.annotations.get(annotationId, Some(version))( fromProtoBytes[AnnotationProto]) ?~> "getAnnotation.failed" @@ -234,7 +238,8 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss version, requestedSkeletonTracingIds, requestedVolumeTracingIds, - requestAll) ?~> "applyUpdates.failed" + requestAll, + reportChangesToWk) ?~> "applyUpdates.failed" } yield updated def getEditableMappingInfo(annotationId: String, tracingId: String, version: Option[Long] = None)( @@ -268,7 +273,8 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss targetVersion: Long, requestedSkeletonTracingIds: List[String], requestedVolumeTracingIds: List[String], - requestAll: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = + requestAll: Boolean, + reportChangesToWk: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = for { updateGroupsAsSaved <- findPendingUpdates(annotationId, annotation.version, targetVersion) ?~> "findPendingUpdates.failed" updatesGroupsRegrouped = regroupByIsolationSensitiveActions(updateGroupsAsSaved) @@ -284,7 +290,10 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss updatesFlat, annotation.version, targetVersion) // TODO: targetVersion should be set per update group - updated <- applyUpdatesGrouped(annotationWithTracingsAndMappings, annotationId, updatesGroupsRegrouped) ?~> "applyUpdates.inner.failed" + updated <- applyUpdatesGrouped(annotationWithTracingsAndMappings, + annotationId, + updatesGroupsRegrouped, + reportChangesToWk) ?~> "applyUpdates.inner.failed" } yield updated private def findEditableMappingsForUpdates( // TODO integrate with findTracings? @@ -390,7 +399,8 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss private def applyUpdatesGrouped( annotation: AnnotationWithTracings, annotationId: String, - updateGroups: List[(Long, List[UpdateAction])] + updateGroups: List[(Long, List[UpdateAction])], + reportChangesToWk: Boolean )(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = { def updateGroupedIter(annotationWithTracingsFox: Fox[AnnotationWithTracings], remainingUpdateGroups: List[(Long, List[UpdateAction])]): Fox[AnnotationWithTracings] = @@ -413,7 +423,8 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss annotation: AnnotationWithTracings, annotationId: String, updates: List[UpdateAction], - targetVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = { + targetVersion: Long, + reportChangesToWk: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = { logger.info(s"applying ${updates.length} to go from v${annotation.version} to v$targetVersion") From ee89ea790d146dd9dc5ab5f1e97c031f8fbac14d Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 23 Oct 2024 10:10:14 +0200 Subject: [PATCH 115/150] fix param, remove addSegmentIndex route --- app/controllers/AnnotationController.scala | 65 ------------------- .../WKRemoteTracingStoreClient.scala | 8 --- conf/webknossos.latest.routes | 1 - .../annotation/TSAnnotationService.scala | 5 +- .../controllers/TSAnnotationController.scala | 18 +---- .../controllers/VolumeTracingController.scala | 26 -------- .../volume/VolumeTracingService.scala | 55 ---------------- ...alableminds.webknossos.tracingstore.routes | 1 - 8 files changed, 6 insertions(+), 173 deletions(-) diff --git a/app/controllers/AnnotationController.scala b/app/controllers/AnnotationController.scala index d5a1943f01f..81c1aa9f332 100755 --- a/app/controllers/AnnotationController.scala +++ b/app/controllers/AnnotationController.scala @@ -25,7 +25,6 @@ import models.task.TaskDAO import models.team.{TeamDAO, TeamService} import models.user.time._ import models.user.{User, UserDAO, UserService} -import net.liftweb.common.Box import play.api.i18n.{Messages, MessagesProvider} import play.api.libs.json._ import play.api.mvc.{Action, AnyContent, PlayBodyParsers} @@ -258,70 +257,6 @@ class AnnotationController @Inject()( } yield result } - def addSegmentIndicesToAll(parallelBatchCount: Int, - dryRun: Boolean, - skipTracings: Option[String]): Action[AnyContent] = - sil.SecuredAction.async { implicit request => - { - for { - _ <- userService.assertIsSuperUser(request.identity._multiUser) ?~> "notAllowed" ~> FORBIDDEN - _ = logger.info("Running migration to add segment index to all volume annotation layers...") - skipTracingsSet = skipTracings.map(_.split(",").toSet).getOrElse(Set()) - _ = if (skipTracingsSet.nonEmpty) { - logger.info(f"Skipping these tracings: ${skipTracingsSet.mkString(",")}") - } - _ = logger.info("Gathering list of volume tracings...") - annotationLayers <- annotationLayerDAO.findAllVolumeLayers - annotationLayersFiltered = annotationLayers.filter(l => !skipTracingsSet.contains(l.tracingId)) - totalCount = annotationLayersFiltered.length - batches = batch(annotationLayersFiltered, parallelBatchCount) - _ = logger.info(f"Processing $totalCount tracings in ${batches.length} batches") - before = Instant.now - results: Seq[List[Box[Unit]]] <- Fox.combined(batches.zipWithIndex.map { - case (batch, index) => addSegmentIndicesToBatch(batch, index, dryRun) - }) - failures = results.flatMap(_.filter(_.isEmpty)) - failureCount: Int = failures.length - successCount: Int = results.map(_.count(_.isDefined)).sum - msg = s"All done (dryRun=$dryRun)! Processed $totalCount tracings in ${batches.length} batches. Took ${Instant - .since(before)}. $failureCount failures, $successCount successes." - _ = if (failures.nonEmpty) { - failures.foreach { failedBox => - logger.info(f"Failed: $failedBox") - } - } - _ = logger.info(msg) - } yield JsonOk(msg) - } - } - - private def addSegmentIndicesToBatch(annotationLayerBatch: List[AnnotationLayer], batchIndex: Int, dryRun: Boolean)( - implicit ec: ExecutionContext) = { - var processedCount = 0 - for { - tracingStore <- tracingStoreDAO.findFirst(GlobalAccessContext) ?~> "tracingStore.notFound" - client = new WKRemoteTracingStoreClient(tracingStore, null, rpc, tracingDataSourceTemporaryStore) - batchCount = annotationLayerBatch.length - results <- Fox.serialSequenceBox(annotationLayerBatch) { annotationLayer => - processedCount += 1 - logger.info( - f"Processing tracing ${annotationLayer.tracingId}. $processedCount of $batchCount in batch $batchIndex (${percent(processedCount, batchCount)})...") - client.addSegmentIndex(annotationLayer.tracingId, dryRun) ?~> s"add segment index failed for ${annotationLayer.tracingId}" - } - _ = logger.info(f"Batch $batchIndex is done. Processed ${annotationLayerBatch.length} tracings.") - } yield results - } - - private def batch[T](allItems: List[T], batchCount: Int): List[List[T]] = { - val batchSize: Int = Math.max(Math.min(allItems.length / batchCount, allItems.length), 1) - allItems.grouped(batchSize).toList - } - - private def percent(done: Int, pending: Int) = { - val value = done.toDouble / pending.toDouble * 100 - f"$value%1.1f %%" - } - private def finishAnnotation(typ: String, id: String, issuingUser: User, timestamp: Instant)( implicit ctx: DBAccessContext): Fox[(Annotation, String)] = for { diff --git a/app/models/annotation/WKRemoteTracingStoreClient.scala b/app/models/annotation/WKRemoteTracingStoreClient.scala index de54a505a10..521baabc1c5 100644 --- a/app/models/annotation/WKRemoteTracingStoreClient.scala +++ b/app/models/annotation/WKRemoteTracingStoreClient.scala @@ -128,14 +128,6 @@ class WKRemoteTracingStoreClient( .postJsonWithJsonResponse[Option[BoundingBox], String](datasetBoundingBox) } - def addSegmentIndex(volumeTracingId: String, dryRun: Boolean): Fox[Unit] = - rpc(s"${tracingStore.url}/tracings/volume/$volumeTracingId/addSegmentIndex").withLongTimeout - .addQueryString("token" -> RpcTokenHolder.webknossosToken) - .addQueryString("dryRun" -> dryRun.toString) - .silent - .post() - .map(_ => ()) - def mergeSkeletonTracingsByIds(tracingIds: List[String], persistTracing: Boolean): Fox[String] = { logger.debug("Called to merge SkeletonTracings by ids." + baseInfo) rpc(s"${tracingStore.url}/tracings/skeleton/mergedFromIds").withLongTimeout diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes index e439b682927..9bd9313e70e 100644 --- a/conf/webknossos.latest.routes +++ b/conf/webknossos.latest.routes @@ -152,7 +152,6 @@ DELETE /annotations/:id POST /annotations/:id/merge/:mergedTyp/:mergedId controllers.AnnotationController.mergeWithoutType(id: String, mergedTyp: String, mergedId: String) GET /annotations/:id/download controllers.AnnotationIOController.downloadWithoutType(id: String, skeletonVersion: Option[Long], volumeVersion: Option[Long], skipVolumeData: Option[Boolean], volumeDataZipFormat: Option[String]) POST /annotations/:id/acquireMutex controllers.AnnotationController.tryAcquiringAnnotationMutex(id: String) -PATCH /annotations/addSegmentIndicesToAll controllers.AnnotationController.addSegmentIndicesToAll(parallelBatchCount: Int, dryRun: Boolean, skipTracings: Option[String]) GET /annotations/:typ/:id/info controllers.AnnotationController.info(typ: String, id: String, timestamp: Option[Long]) PATCH /annotations/:typ/:id/downsample controllers.AnnotationController.downsample(typ: String, id: String, tracingId: String) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 70a285483ce..b617a1cf378 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -410,8 +410,9 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss remainingUpdateGroups match { case List() => Fox.successful(annotationWithTracings) case updateGroup :: tail => - updateGroupedIter(applyUpdates(annotationWithTracings, annotationId, updateGroup._2, updateGroup._1), - tail) + updateGroupedIter( + applyUpdates(annotationWithTracings, annotationId, updateGroup._2, updateGroup._1, reportChangesToWk), + tail) } case _ => annotationWithTracingsFox } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index 755a31aee67..97315646e2c 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -59,7 +59,9 @@ class TSAnnotationController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readAnnotation(annotationId)) { for { newestMaterializableVersion <- annotationService.currentMaterializableVersion(annotationId) - updateLog <- annotationService.updateActionLog(annotationId, newestVersion.getOrElse(newestMaterializableVersion), oldestVersion.getOrElse(0)) + updateLog <- annotationService.updateActionLog(annotationId, + newestVersion.getOrElse(newestMaterializableVersion), + oldestVersion.getOrElse(0)) } yield Ok(updateLog) } } @@ -99,17 +101,3 @@ class TSAnnotationController @Inject()( } } - -// get version history - -// update layer - -// restore of layer - -// delete layer - -// add layer - -// skeleton + volume routes can now take annotationVersion - -// Is an editable mapping a layer? diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index e0c675d94c5..6608d2a14f2 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -2,7 +2,6 @@ package com.scalableminds.webknossos.tracingstore.controllers import com.google.inject.Inject import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int} -import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.ExtendedTypes.ExtendedString import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.VolumeTracing.{VolumeTracing, VolumeTracingOpt, VolumeTracings} @@ -344,31 +343,6 @@ class VolumeTracingController @Inject()( } } - def addSegmentIndex(tracingId: String, dryRun: Boolean): Action[AnyContent] = - Action.async { implicit request => - log() { - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { - for { - annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") - currentVersion <- annotationService.currentMaterializableVersion(tracingId) - before = Instant.now - canAddSegmentIndex <- volumeTracingService.checkIfSegmentIndexMayBeAdded(tracingId, tracing) - processedBucketCountOpt <- Fox.runIf(canAddSegmentIndex)(volumeTracingService - .addSegmentIndex(annotationId, tracingId, tracing, currentVersion, dryRun)) ?~> "addSegmentIndex.failed" - currentVersionNew <- annotationService.currentMaterializableVersion(tracingId) - _ <- Fox.runIf(!dryRun)(bool2Fox( - processedBucketCountOpt.isEmpty || currentVersionNew == currentVersion + 1L) ?~> "Version increment failed. Looks like someone edited the annotation layer in the meantime.") - duration = Instant.since(before) - _ = processedBucketCountOpt.foreach { processedBucketCount => - logger.info( - s"Added segment index (dryRun=$dryRun) for tracing $tracingId. Took $duration for $processedBucketCount buckets") - } - } yield Ok - } - } - } - def requestAdHocMesh(tracingId: String): Action[WebknossosAdHocMeshRequest] = Action.async(validateJson[WebknossosAdHocMeshRequest]) { implicit request => accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 06a9ef08973..25e61a4c287 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -811,61 +811,6 @@ class VolumeTracingService @Inject()( } } - def addSegmentIndex(annotationId: String, - tracingId: String, - tracing: VolumeTracing, - currentVersion: Long, - dryRun: Boolean)(implicit tc: TokenContext): Fox[Option[Int]] = { - var processedBucketCount = 0 - for { - isTemporaryTracing <- isTemporaryTracing(tracingId) - sourceDataLayer = volumeTracingLayer(tracingId, tracing, isTemporaryTracing) - buckets: Iterator[(BucketPosition, Array[Byte])] = sourceDataLayer.bucketProvider.bucketStream() - fallbackLayer <- getFallbackLayer(tracingId, tracing) - mappingName <- selectMappingName(tracing) - segmentIndexBuffer = new VolumeSegmentIndexBuffer(tracingId, - volumeSegmentIndexClient, - currentVersion + 1L, - remoteDatastoreClient, - fallbackLayer, - sourceDataLayer.additionalAxes, - tc) - _ <- Fox.serialCombined(buckets) { - case (bucketPosition, bucketData) => - processedBucketCount += 1 - updateSegmentIndex(segmentIndexBuffer, - bucketPosition, - bucketData, - Empty, - tracing.elementClass, - mappingName, - editableMappingTracingId(tracing, tracingId)) - } - _ <- Fox.runIf(!dryRun)(segmentIndexBuffer.flush()) - updateGroup = UpdateActionGroup( - tracing.version + 1L, - System.currentTimeMillis(), - None, - List(AddSegmentIndexVolumeAction(tracingId)), - None, - None, - "dummyTransactionId", - 1, - 0 - ) - // TODO _ <- Fox.runIf(!dryRun)(handleUpdateGroup(tracingId, updateGroup, tracing.version, userToken)) - } yield Some(processedBucketCount) - } - - def checkIfSegmentIndexMayBeAdded(tracingId: String, tracing: VolumeTracing)(implicit ec: ExecutionContext, - tc: TokenContext): Fox[Boolean] = - for { - fallbackLayerOpt <- Fox.runIf(tracing.fallbackLayer.isDefined)( - remoteFallbackLayerFromVolumeTracing(tracing, tracingId)) - canHaveSegmentIndex <- VolumeSegmentIndexService.canHaveSegmentIndex(remoteDatastoreClient, fallbackLayerOpt) - alreadyHasSegmentIndex = tracing.hasSegmentIndex.getOrElse(false) - } yield canHaveSegmentIndex && !alreadyHasSegmentIndex - def importVolumeData(annotationId: String, tracingId: String, tracing: VolumeTracing, diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index dcb3b41ab8b..df237857e93 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -24,7 +24,6 @@ POST /volume/:tracingId/adHocMesh POST /volume/:tracingId/fullMesh.stl @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.loadFullMeshStl(tracingId: String) POST /volume/:tracingId/segmentIndex/:segmentId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentIndex(tracingId: String, segmentId: Long) POST /volume/:tracingId/importVolumeData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.importVolumeData(tracingId: String) -POST /volume/:tracingId/addSegmentIndex @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.addSegmentIndex(tracingId: String, dryRun: Boolean) GET /volume/:tracingId/findData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.findData(tracingId: String) POST /volume/:tracingId/segmentStatistics/volume @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentVolume(tracingId: String) POST /volume/:tracingId/segmentStatistics/boundingBox @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentBoundingBox(tracingId: String) From a7794d23702ed6a4ee89d1b49fae6e1eeee5bcd1 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 23 Oct 2024 10:19:00 +0200 Subject: [PATCH 116/150] WIP import volume data --- .../controllers/VolumeTracingController.scala | 15 ++++++++------- .../tracings/volume/VolumeTracingService.scala | 11 ++++------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 6608d2a14f2..0b0305f627c 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -22,12 +22,12 @@ import com.scalableminds.webknossos.datastore.models.{ } import com.scalableminds.webknossos.datastore.rpc.RPC import com.scalableminds.webknossos.datastore.services.{FullMeshRequest, UserAccessRequest} -import com.scalableminds.webknossos.tracingstore.annotation.TSAnnotationService +import com.scalableminds.webknossos.tracingstore.annotation.{AnnotationTransactionService, TSAnnotationService} import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService import com.scalableminds.webknossos.tracingstore.tracings.volume.{ - MergedVolumeStats, MagRestrictions, + MergedVolumeStats, TSFullMeshService, VolumeDataZipFormat, VolumeSegmentIndexService, @@ -60,6 +60,7 @@ class VolumeTracingController @Inject()( editableMappingService: EditableMappingService, val slackNotificationService: TSSlackNotificationService, val remoteWebknossosClient: TSRemoteWebknossosClient, + annotationTransactionService: AnnotationTransactionService, volumeSegmentStatisticsService: VolumeSegmentStatisticsService, volumeSegmentIndexService: VolumeSegmentIndexService, fullMeshService: TSFullMeshService, @@ -333,11 +334,11 @@ class VolumeTracingController @Inject()( tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") currentVersion <- request.body.dataParts("currentVersion").headOption.flatMap(_.toIntOpt).toFox zipFile <- request.body.files.headOption.map(f => new File(f.ref.path.toString)).toFox - largestSegmentId <- volumeTracingService.importVolumeData(annotationId, - tracingId, - tracing, - zipFile, - currentVersion) + (updateGroup, largestSegmentId) <- volumeTracingService.importVolumeData(tracingId, + tracing, + zipFile, + currentVersion) + _ <- annotationTransactionService.handleUpdateGroups(annotationId, List(updateGroup)) } yield Ok(Json.toJson(largestSegmentId)) } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 25e61a4c287..1f0d68a805d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -811,11 +811,9 @@ class VolumeTracingService @Inject()( } } - def importVolumeData(annotationId: String, - tracingId: String, - tracing: VolumeTracing, - zipFile: File, - currentVersion: Int)(implicit mp: MessagesProvider, tc: TokenContext): Fox[Long] = + def importVolumeData(tracingId: String, tracing: VolumeTracing, zipFile: File, currentVersion: Int)( + implicit mp: MessagesProvider, + tc: TokenContext): Fox[(UpdateActionGroup, Long)] = if (currentVersion != tracing.version) Fox.failure("version.mismatch") else { @@ -881,8 +879,7 @@ class VolumeTracingService @Inject()( 1, 0 ) - // TODO: _ <- handleUpdateGroup(tracingId, updateGroup, tracing.version, userToken) - } yield mergedVolume.largestSegmentId.toPositiveLong + } yield (updateGroup, mergedVolume.largestSegmentId.toPositiveLong) } } From f150bf26af421197ccdcbf8d89c4bb3d79ff1725 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 23 Oct 2024 11:17:08 +0200 Subject: [PATCH 117/150] sort out some update actions --- .../UpdateGroupHandlingUnitTestSuite.scala | 6 +- .../annotation/AnnotationReversion.scala | 4 +- .../AnnotationTransactionService.scala | 2 +- .../annotation/AnnotationUpdateActions.scala | 108 +++++++++++------- .../annotation/AnnotationWithTracings.scala | 8 +- .../annotation/TSAnnotationService.scala | 22 ++-- .../annotation/UpdateActions.scala | 64 +++++------ .../annotation/UpdateGroupHandling.scala | 4 +- .../EditableMappingUpdater.scala | 4 +- .../updating/SkeletonUpdateActions.scala | 24 +--- .../tracings/volume/VolumeUpdateActions.scala | 25 +--- 11 files changed, 126 insertions(+), 145 deletions(-) diff --git a/test/backend/UpdateGroupHandlingUnitTestSuite.scala b/test/backend/UpdateGroupHandlingUnitTestSuite.scala index dec49103cd1..a2d27f7f00a 100644 --- a/test/backend/UpdateGroupHandlingUnitTestSuite.scala +++ b/test/backend/UpdateGroupHandlingUnitTestSuite.scala @@ -1,7 +1,7 @@ package backend -import com.scalableminds.webknossos.tracingstore.annotation.{RevertToVersionUpdateAction, UpdateGroupHandling} -import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{MergeTreeSkeletonAction} +import com.scalableminds.webknossos.tracingstore.annotation.{RevertToVersionAnnotationAction, UpdateGroupHandling} +import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.MergeTreeSkeletonAction import org.scalatestplus.play.PlaySpec class UpdateGroupHandlingUnitTestSuite extends PlaySpec with UpdateGroupHandling { @@ -16,7 +16,7 @@ class UpdateGroupHandlingUnitTestSuite extends PlaySpec with UpdateGroupHandling )), (6L, List( - RevertToVersionUpdateAction(sourceVersion = 1), + RevertToVersionAnnotationAction(sourceVersion = 1), )), (7L, List( diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala index b85e053366d..f7108ffbaad 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala @@ -14,7 +14,7 @@ trait AnnotationReversion { def revertDistributedElements(annotationId: String, currentAnnotationWithTracings: AnnotationWithTracings, sourceAnnotationWithTracings: AnnotationWithTracings, - revertAction: RevertToVersionUpdateAction, + revertAction: RevertToVersionAnnotationAction, newVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Unit] = // TODO segment index, volume buckets, proofreading data for { @@ -36,7 +36,7 @@ trait AnnotationReversion { } yield () private def revertEditableMappingFields(currentAnnotationWithTracings: AnnotationWithTracings, - revertAction: RevertToVersionUpdateAction, + revertAction: RevertToVersionAnnotationAction, tracingId: String)(implicit ec: ExecutionContext): Fox[Unit] = for { updater <- currentAnnotationWithTracings.getEditableMappingUpdater(tracingId).toFox diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala index 2399ed2549e..2bad1a3e045 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala @@ -229,7 +229,7 @@ class AnnotationTransactionService @Inject()(handledGroupIdStore: TracingStoreRe } actionsWithInfo.map { case a: UpdateBucketVolumeAction => a.withoutBase64Data - case a: AddLayerAnnotationUpdateAction => a.copy(tracingId = Some(TracingId.generate)) + case a: AddLayerAnnotationAction => a.copy(tracingId = Some(TracingId.generate)) case a => a } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala index d000402ef0a..4d44bcc1407 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala @@ -18,14 +18,15 @@ object AnnotationLayerParameters { Json.using[WithDefaultValues].format[AnnotationLayerParameters] } -trait AnnotationUpdateAction extends ApplyImmediatelyUpdateAction +trait AnnotationUpdateAction extends UpdateAction -case class AddLayerAnnotationUpdateAction(layerParameters: AnnotationLayerParameters, - tracingId: Option[String] = None, // filled in by backend eagerly on save - actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) - extends AnnotationUpdateAction { +case class AddLayerAnnotationAction(layerParameters: AnnotationLayerParameters, + tracingId: Option[String] = None, // filled in by backend eagerly on save + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) + extends AnnotationUpdateAction + with ApplyImmediatelyUpdateAction { override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) @@ -33,13 +34,14 @@ case class AddLayerAnnotationUpdateAction(layerParameters: AnnotationLayerParame this.copy(actionAuthorId = authorId) } -case class DeleteLayerAnnotationUpdateAction(tracingId: String, - layerName: String, // Just stored for nicer-looking history - `type`: AnnotationLayerType, // Just stored for nicer-looking history - actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) - extends AnnotationUpdateAction { +case class DeleteLayerAnnotationAction(tracingId: String, + layerName: String, // Just stored for nicer-looking history + `type`: AnnotationLayerType, // Just stored for nicer-looking history + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) + extends AnnotationUpdateAction + with ApplyImmediatelyUpdateAction { override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) @@ -47,12 +49,13 @@ case class DeleteLayerAnnotationUpdateAction(tracingId: String, this.copy(actionAuthorId = authorId) } -case class UpdateLayerMetadataAnnotationUpdateAction(tracingId: String, - layerName: String, - actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) - extends AnnotationUpdateAction { +case class UpdateLayerMetadataAnnotationAction(tracingId: String, + layerName: String, + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) + extends AnnotationUpdateAction + with ApplyImmediatelyUpdateAction { override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) @@ -60,12 +63,13 @@ case class UpdateLayerMetadataAnnotationUpdateAction(tracingId: String, this.copy(actionAuthorId = authorId) } -case class UpdateMetadataAnnotationUpdateAction(name: Option[String], - description: Option[String], - actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) - extends AnnotationUpdateAction { +case class UpdateMetadataAnnotationAction(name: Option[String], + description: Option[String], + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) + extends AnnotationUpdateAction + with ApplyImmediatelyUpdateAction { override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) @@ -73,33 +77,51 @@ case class UpdateMetadataAnnotationUpdateAction(name: Option[String], this.copy(actionAuthorId = authorId) } -case class RevertToVersionUpdateAction(sourceVersion: Long, - actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) +case class RevertToVersionAnnotationAction(sourceVersion: Long, + actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) + extends AnnotationUpdateAction + with ApplyImmediatelyUpdateAction { + override def addTimestamp(timestamp: Long): UpdateAction = + this.copy(actionTimestamp = Some(timestamp)) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def addAuthorId(authorId: Option[String]): UpdateAction = + this.copy(actionAuthorId = authorId) +} + +case class UpdateTdCameraAnnotationAction(actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) extends AnnotationUpdateAction { + override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + + override def isViewOnlyChange: Boolean = true } -object AddLayerAnnotationUpdateAction { - implicit val jsonFormat: OFormat[AddLayerAnnotationUpdateAction] = Json.format[AddLayerAnnotationUpdateAction] +object AddLayerAnnotationAction { + implicit val jsonFormat: OFormat[AddLayerAnnotationAction] = Json.format[AddLayerAnnotationAction] +} +object DeleteLayerAnnotationAction { + implicit val jsonFormat: OFormat[DeleteLayerAnnotationAction] = Json.format[DeleteLayerAnnotationAction] } -object DeleteLayerAnnotationUpdateAction { - implicit val jsonFormat: OFormat[DeleteLayerAnnotationUpdateAction] = Json.format[DeleteLayerAnnotationUpdateAction] +object UpdateLayerMetadataAnnotationAction { + implicit val jsonFormat: OFormat[UpdateLayerMetadataAnnotationAction] = + Json.format[UpdateLayerMetadataAnnotationAction] } -object UpdateLayerMetadataAnnotationUpdateAction { - implicit val jsonFormat: OFormat[UpdateLayerMetadataAnnotationUpdateAction] = - Json.format[UpdateLayerMetadataAnnotationUpdateAction] +object UpdateMetadataAnnotationAction { + implicit val jsonFormat: OFormat[UpdateMetadataAnnotationAction] = + Json.format[UpdateMetadataAnnotationAction] } -object UpdateMetadataAnnotationUpdateAction { - implicit val jsonFormat: OFormat[UpdateMetadataAnnotationUpdateAction] = - Json.format[UpdateMetadataAnnotationUpdateAction] +object RevertToVersionAnnotationAction { + implicit val jsonFormat: OFormat[RevertToVersionAnnotationAction] = + Json.format[RevertToVersionAnnotationAction] } -object RevertToVersionUpdateAction { - implicit val jsonFormat: OFormat[RevertToVersionUpdateAction] = - Json.format[RevertToVersionUpdateAction] +object UpdateTdCameraAnnotationAction { + implicit val jsonFormat: OFormat[UpdateTdCameraAnnotationAction] = Json.format[UpdateTdCameraAnnotationAction] } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index 6996b398d6d..fddeda5bf6b 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -78,7 +78,7 @@ case class AnnotationWithTracings( def version: Long = annotation.version - def addLayer(a: AddLayerAnnotationUpdateAction, + def addLayer(a: AddLayerAnnotationAction, tracingId: String, tracing: Either[SkeletonTracing, VolumeTracing]): AnnotationWithTracings = this.copy( @@ -91,14 +91,14 @@ case class AnnotationWithTracings( tracingsById = tracingsById.updated(tracingId, tracing) ) - def deleteTracing(a: DeleteLayerAnnotationUpdateAction): AnnotationWithTracings = + def deleteTracing(a: DeleteLayerAnnotationAction): AnnotationWithTracings = this.copy(annotation = annotation.copy(layers = annotation.layers.filter(_.tracingId != a.tracingId))) - def updateLayerMetadata(a: UpdateLayerMetadataAnnotationUpdateAction): AnnotationWithTracings = + def updateLayerMetadata(a: UpdateLayerMetadataAnnotationAction): AnnotationWithTracings = this.copy(annotation = annotation.copy(layers = annotation.layers.map(l => if (l.tracingId == a.tracingId) l.copy(name = a.layerName) else l))) - def updateMetadata(a: UpdateMetadataAnnotationUpdateAction): AnnotationWithTracings = + def updateMetadata(a: UpdateMetadataAnnotationAction): AnnotationWithTracings = this.copy(annotation = annotation.copy(name = a.name, description = a.description)) def withVersion(newVersion: Long): AnnotationWithTracings = { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index b617a1cf378..1240e66c9bf 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -106,13 +106,13 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss )(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = for { updated <- updateAction match { - case a: AddLayerAnnotationUpdateAction => + case a: AddLayerAnnotationAction => addLayer(annotationId, annotationWithTracings, a, targetVersion) - case a: DeleteLayerAnnotationUpdateAction => + case a: DeleteLayerAnnotationAction => Fox.successful(annotationWithTracings.deleteTracing(a)) - case a: UpdateLayerMetadataAnnotationUpdateAction => + case a: UpdateLayerMetadataAnnotationAction => Fox.successful(annotationWithTracings.updateLayerMetadata(a)) - case a: UpdateMetadataAnnotationUpdateAction => + case a: UpdateMetadataAnnotationAction => Fox.successful(annotationWithTracings.updateMetadata(a)) case a: SkeletonUpdateAction => annotationWithTracings.applySkeletonAction(a) ?~> "applySkeletonAction.failed" @@ -125,7 +125,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss annotationWithTracings.applyVolumeAction(a) case a: EditableMappingUpdateAction => annotationWithTracings.applyEditableMappingAction(a) - case a: RevertToVersionUpdateAction => + case a: RevertToVersionAnnotationAction => revertToVersion(annotationId, annotationWithTracings, a, targetVersion) // TODO if the revert action is not isolated, we need not the target version of all but the target version of this update case _: BucketMutatingVolumeUpdateAction => Fox.successful(annotationWithTracings) // No-op, as bucket-mutating actions are performed eagerly, so not here. @@ -135,7 +135,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss private def addLayer(annotationId: String, annotationWithTracings: AnnotationWithTracings, - action: AddLayerAnnotationUpdateAction, + action: AddLayerAnnotationAction, targetVersion: Long)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { tracingId <- action.tracingId.toFox ?~> "add layer action has no tracingId" @@ -146,10 +146,10 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } yield updated private def revertToVersion( - annotationId: String, - annotationWithTracings: AnnotationWithTracings, - revertAction: RevertToVersionUpdateAction, - newVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = + annotationId: String, + annotationWithTracings: AnnotationWithTracings, + revertAction: RevertToVersionAnnotationAction, + newVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = // Note: works only after “ironing out” the update action groups // TODO: read old annotationProto, tracing, buckets, segment indeces for { @@ -164,7 +164,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss _ <- revertDistributedElements(annotationId, annotationWithTracings, sourceAnnotation, revertAction, newVersion) } yield sourceAnnotation - def createTracing(a: AddLayerAnnotationUpdateAction)( + def createTracing(a: AddLayerAnnotationAction)( implicit ec: ExecutionContext): Fox[Either[SkeletonTracing, VolumeTracing]] = Fox.failure("not implemented") // TODO create tracing object (ask wk for needed parameters e.g. fallback layer info?) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala index b13a71c58db..091ae7f327e 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala @@ -14,7 +14,6 @@ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{ MergeTreeSkeletonAction, MoveTreeComponentSkeletonAction, UpdateNodeSkeletonAction, - UpdateTdCameraSkeletonAction, UpdateTracingSkeletonAction, UpdateTreeEdgesVisibilitySkeletonAction, UpdateTreeGroupVisibilitySkeletonAction, @@ -35,7 +34,6 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ UpdateMappingNameVolumeAction, UpdateSegmentGroupsVolumeAction, UpdateSegmentVolumeAction, - UpdateTdCameraVolumeAction, UpdateTracingVolumeAction, UpdateUserBoundingBoxVisibilityVolumeAction, UpdateUserBoundingBoxesVolumeAction @@ -90,30 +88,29 @@ object UpdateAction { case "updateBucket" => deserialize[UpdateBucketVolumeAction](jsonValue) case "updateVolumeTracing" => deserialize[UpdateTracingVolumeAction](jsonValue) case "updateUserBoundingBoxesInVolumeTracing" => - deserialize[UpdateUserBoundingBoxesVolumeAction](jsonValue) // TODO: rename key (must be different from skeleton action) + deserialize[UpdateUserBoundingBoxesVolumeAction](jsonValue) case "updateUserBoundingBoxVisibilityInVolumeTracing" => deserialize[UpdateUserBoundingBoxVisibilityVolumeAction](jsonValue) - case "removeFallbackLayer" => deserialize[RemoveFallbackLayerVolumeAction](jsonValue) - case "importVolumeTracing" => deserialize[ImportVolumeDataVolumeAction](jsonValue) - case "updateTdCameraSkeleton" => deserialize[UpdateTdCameraSkeletonAction](jsonValue) // TODO deduplicate? - case "updateTdCameraVolume" => deserialize[UpdateTdCameraVolumeAction](jsonValue) - case "createSegment" => deserialize[CreateSegmentVolumeAction](jsonValue) - case "updateSegment" => deserialize[UpdateSegmentVolumeAction](jsonValue) - case "updateSegmentGroups" => deserialize[UpdateSegmentGroupsVolumeAction](jsonValue) - case "deleteSegment" => deserialize[DeleteSegmentVolumeAction](jsonValue) - case "deleteSegmentData" => deserialize[DeleteSegmentDataVolumeAction](jsonValue) - case "updateMappingName" => deserialize[UpdateMappingNameVolumeAction](jsonValue) + case "removeFallbackLayer" => deserialize[RemoveFallbackLayerVolumeAction](jsonValue) + case "importVolumeTracing" => deserialize[ImportVolumeDataVolumeAction](jsonValue) + case "createSegment" => deserialize[CreateSegmentVolumeAction](jsonValue) + case "updateSegment" => deserialize[UpdateSegmentVolumeAction](jsonValue) + case "updateSegmentGroups" => deserialize[UpdateSegmentGroupsVolumeAction](jsonValue) + case "deleteSegment" => deserialize[DeleteSegmentVolumeAction](jsonValue) + case "deleteSegmentData" => deserialize[DeleteSegmentDataVolumeAction](jsonValue) + case "updateMappingName" => deserialize[UpdateMappingNameVolumeAction](jsonValue) // Editable Mapping case "mergeAgglomerate" => deserialize[MergeAgglomerateUpdateAction](jsonValue) case "splitAgglomerate" => deserialize[SplitAgglomerateUpdateAction](jsonValue) // Annotation - case "addLayerToAnnotation" => deserialize[AddLayerAnnotationUpdateAction](jsonValue) - case "deleteLayerFromAnnotation" => deserialize[DeleteLayerAnnotationUpdateAction](jsonValue) - case "updateLayerMetadata" => deserialize[UpdateLayerMetadataAnnotationUpdateAction](jsonValue) - case "updateMetadataOfAnnotation" => deserialize[UpdateMetadataAnnotationUpdateAction](jsonValue) - case "revertToVersion" => deserialize[RevertToVersionUpdateAction](jsonValue) + case "addLayerToAnnotation" => deserialize[AddLayerAnnotationAction](jsonValue) + case "deleteLayerFromAnnotation" => deserialize[DeleteLayerAnnotationAction](jsonValue) + case "updateLayerMetadata" => deserialize[UpdateLayerMetadataAnnotationAction](jsonValue) + case "updateMetadataOfAnnotation" => deserialize[UpdateMetadataAnnotationAction](jsonValue) + case "revertToVersion" => deserialize[RevertToVersionAnnotationAction](jsonValue) + case "updateTdCamera" => deserialize[UpdateTdCameraAnnotationAction](jsonValue) case unknownAction: String => JsError(s"Invalid update action s'$unknownAction'") } @@ -170,8 +167,6 @@ object UpdateAction { case s: UpdateUserBoundingBoxVisibilitySkeletonAction => Json.obj("name" -> "updateUserBoundingBoxVisibilityInSkeletonTracing", "value" -> Json.toJson(s)(UpdateUserBoundingBoxVisibilitySkeletonAction.jsonFormat)) - case s: UpdateTdCameraSkeletonAction => - Json.obj("name" -> "updateTdCameraSkeleton", "value" -> Json.toJson(s)(UpdateTdCameraSkeletonAction.jsonFormat)) // Volume case s: UpdateBucketVolumeAction => @@ -188,8 +183,6 @@ object UpdateAction { Json.obj("name" -> "removeFallbackLayer", "value" -> Json.toJson(s)(RemoveFallbackLayerVolumeAction.jsonFormat)) case s: ImportVolumeDataVolumeAction => Json.obj("name" -> "importVolumeTracing", "value" -> Json.toJson(s)(ImportVolumeDataVolumeAction.jsonFormat)) - case s: UpdateTdCameraVolumeAction => - Json.obj("name" -> "updateTdCameraVolume", "value" -> Json.toJson(s)(UpdateTdCameraVolumeAction.jsonFormat)) case s: CreateSegmentVolumeAction => Json.obj("name" -> "createSegment", "value" -> Json.toJson(s)(CreateSegmentVolumeAction.jsonFormat)) case s: UpdateSegmentVolumeAction => @@ -209,19 +202,21 @@ object UpdateAction { Json.obj("name" -> "mergeAgglomerate", "value" -> Json.toJson(s)(MergeAgglomerateUpdateAction.jsonFormat)) // Annotation - case s: AddLayerAnnotationUpdateAction => - Json.obj("name" -> "addLayerToAnnotation", "value" -> Json.toJson(s)(AddLayerAnnotationUpdateAction.jsonFormat)) - case s: DeleteLayerAnnotationUpdateAction => + case s: AddLayerAnnotationAction => + Json.obj("name" -> "addLayerToAnnotation", "value" -> Json.toJson(s)(AddLayerAnnotationAction.jsonFormat)) + case s: DeleteLayerAnnotationAction => Json.obj("name" -> "deleteLayerFromAnnotation", - "value" -> Json.toJson(s)(DeleteLayerAnnotationUpdateAction.jsonFormat)) - case s: UpdateLayerMetadataAnnotationUpdateAction => + "value" -> Json.toJson(s)(DeleteLayerAnnotationAction.jsonFormat)) + case s: UpdateLayerMetadataAnnotationAction => Json.obj("name" -> "updateLayerMetadata", - "value" -> Json.toJson(s)(UpdateLayerMetadataAnnotationUpdateAction.jsonFormat)) - case s: UpdateMetadataAnnotationUpdateAction => + "value" -> Json.toJson(s)(UpdateLayerMetadataAnnotationAction.jsonFormat)) + case s: UpdateMetadataAnnotationAction => Json.obj("name" -> "updateMetadataOfAnnotation", - "value" -> Json.toJson(s)(UpdateMetadataAnnotationUpdateAction.jsonFormat)) - case s: RevertToVersionUpdateAction => - Json.obj("name" -> "revertToVersion", "value" -> Json.toJson(s)(RevertToVersionUpdateAction.jsonFormat)) + "value" -> Json.toJson(s)(UpdateMetadataAnnotationAction.jsonFormat)) + case s: RevertToVersionAnnotationAction => + Json.obj("name" -> "revertToVersion", "value" -> Json.toJson(s)(RevertToVersionAnnotationAction.jsonFormat)) + case s: UpdateTdCameraAnnotationAction => + Json.obj("name" -> "updateTdCamera", "value" -> Json.toJson(s)(UpdateTdCameraAnnotationAction.jsonFormat)) } } } @@ -236,9 +231,8 @@ case class UpdateActionGroup(version: Long, transactionGroupCount: Int, transactionGroupIndex: Int) { - def significantChangesCount: Int = 1 // TODO - - def viewChangesCount: Int = 1 // TODO + def significantChangesCount: Int = actions.count(!_.isViewOnlyChange) + def viewChangesCount: Int = actions.count(_.isViewOnlyChange) } object UpdateActionGroup { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateGroupHandling.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateGroupHandling.scala index 6893d1bcdc5..de77458a9e3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateGroupHandling.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateGroupHandling.scala @@ -25,8 +25,8 @@ trait UpdateGroupHandling { } private def isIsolationSensitiveAction(a: UpdateAction): Boolean = a match { - case _: RevertToVersionUpdateAction => true - case _: AddLayerAnnotationUpdateAction => true + case _: RevertToVersionAnnotationAction => true + case _: AddLayerAnnotationAction => true case _ => false } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index 15f5fda72cc..d7231195a67 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -11,7 +11,7 @@ import com.scalableminds.webknossos.datastore.SegmentToAgglomerateProto.{ } import com.scalableminds.webknossos.tracingstore.TSRemoteDatastoreClient import com.scalableminds.webknossos.tracingstore.annotation.{ - RevertToVersionUpdateAction, + RevertToVersionAnnotationAction, TSAnnotationService, UpdateAction } @@ -418,7 +418,7 @@ class EditableMappingUpdater( ) } - def revertToVersion(revertAction: RevertToVersionUpdateAction)(implicit ec: ExecutionContext): Fox[Unit] = + def revertToVersion(revertAction: RevertToVersionAnnotationAction)(implicit ec: ExecutionContext): Fox[Unit] = for { _ <- bool2Fox(revertAction.sourceVersion <= oldVersion) ?~> "trying to revert editable mapping to a version not yet present in the database" _ = segmentToAgglomerateBuffer.clear() diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala index f78b744f3fa..4b7a79a4be8 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala @@ -402,6 +402,8 @@ case class UpdateTracingSkeletonAction(activeNode: Option[Int], override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + + override def isViewOnlyChange: Boolean = true } case class UpdateTreeVisibilitySkeletonAction(treeId: Int, @@ -425,6 +427,8 @@ case class UpdateTreeVisibilitySkeletonAction(treeId: Int, override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + + override def isViewOnlyChange: Boolean = true } case class UpdateTreeGroupVisibilitySkeletonAction(treeGroupId: Option[Int], @@ -534,23 +538,8 @@ case class UpdateUserBoundingBoxVisibilitySkeletonAction(boundingBoxId: Option[I override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) -} - -case class UpdateTdCameraSkeletonAction(actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - actionTracingId: String, - info: Option[String] = None) - extends SkeletonUpdateAction { - - override def applyOn(tracing: SkeletonTracing): SkeletonTracing = tracing - - override def addTimestamp(timestamp: Long): UpdateAction = - this.copy(actionTimestamp = Some(timestamp)) - - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction = - this.copy(actionAuthorId = authorId) + override def isViewOnlyChange: Boolean = true } object CreateTreeSkeletonAction { @@ -608,6 +597,3 @@ object UpdateUserBoundingBoxVisibilitySkeletonAction { implicit val jsonFormat: OFormat[UpdateUserBoundingBoxVisibilitySkeletonAction] = Json.format[UpdateUserBoundingBoxVisibilitySkeletonAction] } -object UpdateTdCameraSkeletonAction { - implicit val jsonFormat: OFormat[UpdateTdCameraSkeletonAction] = Json.format[UpdateTdCameraSkeletonAction] -} diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala index 04833f3dbcc..a3e7ff236b2 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala @@ -112,8 +112,6 @@ case class UpdateUserBoundingBoxVisibilityVolumeAction(boundingBoxId: Option[Int this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def isViewOnlyChange: Boolean = true - override def applyOn(tracing: VolumeTracing): VolumeTracing = { def updateUserBoundingBoxes(): Seq[NamedBoundingBoxProto] = @@ -126,6 +124,8 @@ case class UpdateUserBoundingBoxVisibilityVolumeAction(boundingBoxId: Option[Int tracing.withUserBoundingBoxes(updateUserBoundingBoxes()) } + + override def isViewOnlyChange: Boolean = true } case class RemoveFallbackLayerVolumeAction(actionTracingId: String, @@ -172,24 +172,6 @@ case class AddSegmentIndexVolumeAction(actionTracingId: String, } -case class UpdateTdCameraVolumeAction(actionTracingId: String, - actionTimestamp: Option[Long] = None, - actionAuthorId: Option[String] = None, - info: Option[String] = None) - extends ApplyableVolumeUpdateAction { - - override def addTimestamp(timestamp: Long): VolumeUpdateAction = - this.copy(actionTimestamp = Some(timestamp)) - override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = - this.copy(actionAuthorId = authorId) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - - override def applyOn(tracing: VolumeTracing): VolumeTracing = - tracing - - override def isViewOnlyChange: Boolean = true -} - case class CreateSegmentVolumeAction(id: Long, anchorPosition: Option[Vec3Int], name: Option[String], @@ -393,9 +375,6 @@ object ImportVolumeDataVolumeAction { object AddSegmentIndexVolumeAction { implicit val jsonFormat: OFormat[AddSegmentIndexVolumeAction] = Json.format[AddSegmentIndexVolumeAction] } -object UpdateTdCameraVolumeAction { - implicit val jsonFormat: OFormat[UpdateTdCameraVolumeAction] = Json.format[UpdateTdCameraVolumeAction] -} object CreateSegmentVolumeAction { implicit val jsonFormat: OFormat[CreateSegmentVolumeAction] = Json.format[CreateSegmentVolumeAction] } From ecb4b9d443eec597f0a148cdc4ee513fe4adcf8b Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 24 Oct 2024 11:32:59 +0200 Subject: [PATCH 118/150] prepare duplicate --- app/controllers/AnnotationController.scala | 59 ++++--------------- app/models/annotation/AnnotationService.scala | 4 +- .../WKRemoteTracingStoreClient.scala | 35 +++++++---- conf/webknossos.latest.routes | 2 - .../webknossos/datastore/rpc/RPCRequest.scala | 5 ++ .../annotation/TSAnnotationService.scala | 9 +-- .../controllers/TSAnnotationController.scala | 20 +++++++ ...alableminds.webknossos.tracingstore.routes | 1 + 8 files changed, 66 insertions(+), 69 deletions(-) diff --git a/app/controllers/AnnotationController.scala b/app/controllers/AnnotationController.scala index 81c1aa9f332..ff05b2b9cc9 100755 --- a/app/controllers/AnnotationController.scala +++ b/app/controllers/AnnotationController.scala @@ -3,7 +3,6 @@ package controllers import org.apache.pekko.util.Timeout import play.silhouette.api.Silhouette import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext} -import com.scalableminds.util.geometry.BoundingBox import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.datastore.models.annotation.{ @@ -11,8 +10,8 @@ import com.scalableminds.webknossos.datastore.models.annotation.{ AnnotationLayerStatistics, AnnotationLayerType } -import com.scalableminds.webknossos.datastore.rpc.RPC import com.scalableminds.webknossos.tracingstore.annotation.AnnotationLayerParameters +import com.scalableminds.webknossos.tracingstore.tracings.volume.MagRestrictions import com.scalableminds.webknossos.tracingstore.tracings.{TracingId, TracingType} import mail.{MailchimpClient, MailchimpTag} import models.analytics.{AnalyticsService, CreateAnnotationEvent, OpenAnnotationEvent} @@ -43,7 +42,6 @@ class AnnotationController @Inject()( userDAO: UserDAO, organizationDAO: OrganizationDAO, datasetDAO: DatasetDAO, - tracingStoreDAO: TracingStoreDAO, datasetService: DatasetService, annotationService: AnnotationService, annotationMutexService: AnnotationMutexService, @@ -59,9 +57,7 @@ class AnnotationController @Inject()( analyticsService: AnalyticsService, slackNotificationService: SlackNotificationService, mailchimpClient: MailchimpClient, - tracingDataSourceTemporaryStore: TracingDataSourceTemporaryStore, conf: WkConf, - rpc: RPC, sil: Silhouette[WkEnv])(implicit ec: ExecutionContext, bodyParsers: PlayBodyParsers) extends Controller with UserAwareRequestLogging @@ -233,30 +229,6 @@ class AnnotationController @Inject()( } yield JsonOk(json) } - def downsample(typ: String, id: String, tracingId: String): Action[AnyContent] = sil.SecuredAction.async { - implicit request => - for { - _ <- bool2Fox(AnnotationType.Explorational.toString == typ) ?~> "annotation.downsample.explorationalsOnly" - restrictions <- provider.restrictionsFor(typ, id) ?~> "restrictions.notFound" ~> NOT_FOUND - _ <- restrictions.allowUpdate(request.identity) ?~> "notAllowed" ~> FORBIDDEN - annotation <- provider.provideAnnotation(typ, id, request.identity) - annotationLayer <- annotation.annotationLayers - .find(_.tracingId == tracingId) - .toFox ?~> "annotation.downsample.layerNotFound" - _ <- annotationService.downsampleAnnotation(annotation, annotationLayer) ?~> "annotation.downsample.failed" - updated <- provider.provideAnnotation(typ, id, request.identity) - json <- annotationService.publicWrites(updated, Some(request.identity)) ?~> "annotation.write.failed" - } yield JsonOk(json) - } - - def downsampleWithoutType(id: String, tracingId: String): Action[AnyContent] = sil.SecuredAction.async { - implicit request => - for { - annotation <- provider.provideAnnotation(id, request.identity) ~> NOT_FOUND - result <- downsample(annotation.typ.toString, id, tracingId)(request) - } yield result - } - private def finishAnnotation(typ: String, id: String, issuingUser: User, timestamp: Instant)( implicit ctx: DBAccessContext): Fox[(Annotation, String)] = for { @@ -461,12 +433,16 @@ class AnnotationController @Inject()( datasetService.dataSourceFor(dataset).flatMap(_.toUsable).map(Some(_)) else Fox.successful(None) tracingStoreClient <- tracingStoreService.clientFor(dataset) - newAnnotationLayers <- Fox.serialCombined(annotation.annotationLayers) { annotationLayer => - duplicateAnnotationLayer(annotationLayer, - annotation._task.isDefined, - dataSource.map(_.boundingBox), - tracingStoreClient) - } + newAnnotationProto <- tracingStoreClient.duplicateAnnotation( + annotation._id, + version = None, + isFromTask = annotation._task.isDefined, + editPosition = None, + editRotation = None, + boundingBox = dataSource.map(_.boundingBox), + magRestrictions = MagRestrictions.empty + ) + newAnnotationLayers = newAnnotationProto.layers.map(AnnotationLayer.fromProto) clonedAnnotation <- annotationService.createFrom(user, dataset, newAnnotationLayers, @@ -475,19 +451,6 @@ class AnnotationController @Inject()( annotation.description) ?~> Messages("annotation.create.failed") } yield clonedAnnotation - private def duplicateAnnotationLayer(annotationLayer: AnnotationLayer, - isFromTask: Boolean, - datasetBoundingBox: Option[BoundingBox], - tracingStoreClient: WKRemoteTracingStoreClient): Fox[AnnotationLayer] = - for { - - newTracingId <- if (annotationLayer.typ == AnnotationLayerType.Skeleton) { - tracingStoreClient.duplicateSkeletonTracing(annotationLayer.tracingId, None, isFromTask) ?~> "Failed to duplicate skeleton tracing." - } else { - tracingStoreClient.duplicateVolumeTracing(annotationLayer.tracingId, isFromTask, datasetBoundingBox) ?~> "Failed to duplicate volume tracing." - } - } yield annotationLayer.copy(tracingId = newTracingId) - def tryAcquiringAnnotationMutex(id: String): Action[AnyContent] = sil.SecuredAction.async { implicit request => logTime(slackNotificationService.noticeSlowRequest, durationThreshold = 1 second) { diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index de3160bc87c..5c7cc8aa431 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -502,7 +502,7 @@ class AnnotationService @Inject()( def createFrom(user: User, dataset: Dataset, - annotationLayers: List[AnnotationLayer], + annotationLayers: Seq[AnnotationLayer], annotationType: AnnotationType, name: Option[String], description: String): Fox[Annotation] = @@ -513,7 +513,7 @@ class AnnotationService @Inject()( None, teamId, user._id, - annotationLayers, + annotationLayers.toList, description, name = name.getOrElse(""), typ = annotationType) diff --git a/app/models/annotation/WKRemoteTracingStoreClient.scala b/app/models/annotation/WKRemoteTracingStoreClient.scala index 521baabc1c5..017cc421d96 100644 --- a/app/models/annotation/WKRemoteTracingStoreClient.scala +++ b/app/models/annotation/WKRemoteTracingStoreClient.scala @@ -90,23 +90,34 @@ class WKRemoteTracingStoreClient( .postProto[AnnotationProto](annotationProto) } - def duplicateSkeletonTracing(skeletonTracingId: String, - versionString: Option[String] = None, - isFromTask: Boolean = false, - editPosition: Option[Vec3Int] = None, - editRotation: Option[Vec3Double] = None, - boundingBox: Option[BoundingBox] = None): Fox[String] = { - logger.debug("Called to duplicate SkeletonTracing." + baseInfo) - rpc(s"${tracingStore.url}/tracings/skeleton/$skeletonTracingId/duplicate").withLongTimeout + def duplicateAnnotation(annotationId: ObjectId, + version: Option[Long], + isFromTask: Boolean, + editPosition: Option[Vec3Int], + editRotation: Option[Vec3Double], + boundingBox: Option[BoundingBox], + magRestrictions: MagRestrictions, + ): Fox[AnnotationProto] = { + logger.debug(s"Called to duplicate annotation $annotationId." + baseInfo) + rpc(s"${tracingStore.url}/tracings/annotation/$annotationId/duplicate").withLongTimeout .addQueryString("token" -> RpcTokenHolder.webknossosToken) - .addQueryStringOptional("version", versionString) + .addQueryStringOptional("version", version.map(_.toString)) .addQueryStringOptional("editPosition", editPosition.map(_.toUriLiteral)) .addQueryStringOptional("editRotation", editRotation.map(_.toUriLiteral)) .addQueryStringOptional("boundingBox", boundingBox.map(_.toLiteral)) - .addQueryString("fromTask" -> isFromTask.toString) - .postWithJsonResponse[String] + .addQueryString("isFromTask" -> isFromTask.toString) + .addQueryStringOptional("minMag", magRestrictions.minStr) + .addQueryStringOptional("maxMag", magRestrictions.maxStr) + .postWithProtoResponse[AnnotationProto]()(AnnotationProto) } + def duplicateSkeletonTracing(skeletonTracingId: String, + versionString: Option[String] = None, + isFromTask: Boolean = false, + editPosition: Option[Vec3Int] = None, + editRotation: Option[Vec3Double] = None, + boundingBox: Option[BoundingBox] = None): Fox[String] = ??? + def duplicateVolumeTracing(volumeTracingId: String, isFromTask: Boolean = false, datasetBoundingBox: Option[BoundingBox] = None, @@ -119,8 +130,6 @@ class WKRemoteTracingStoreClient( rpc(s"${tracingStore.url}/tracings/volume/$volumeTracingId/duplicate").withLongTimeout .addQueryString("token" -> RpcTokenHolder.webknossosToken) .addQueryString("fromTask" -> isFromTask.toString) - .addQueryStringOptional("minMag", magRestrictions.minStr) - .addQueryStringOptional("maxMag", magRestrictions.maxStr) .addQueryStringOptional("editPosition", editPosition.map(_.toUriLiteral)) .addQueryStringOptional("editRotation", editRotation.map(_.toUriLiteral)) .addQueryStringOptional("boundingBox", boundingBox.map(_.toLiteral)) diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes index 9bd9313e70e..877d9e1d28a 100644 --- a/conf/webknossos.latest.routes +++ b/conf/webknossos.latest.routes @@ -147,14 +147,12 @@ PATCH /annotations/:typ/:id/transfer PATCH /annotations/:typ/:id/editLockedState controllers.AnnotationController.editLockedState(typ: String, id: String, isLockedByOwner: Boolean) GET /annotations/:id/info controllers.AnnotationController.infoWithoutType(id: String, timestamp: Option[Long]) -PATCH /annotations/:id/downsample controllers.AnnotationController.downsampleWithoutType(id: String, tracingId: String) DELETE /annotations/:id controllers.AnnotationController.cancelWithoutType(id: String) POST /annotations/:id/merge/:mergedTyp/:mergedId controllers.AnnotationController.mergeWithoutType(id: String, mergedTyp: String, mergedId: String) GET /annotations/:id/download controllers.AnnotationIOController.downloadWithoutType(id: String, skeletonVersion: Option[Long], volumeVersion: Option[Long], skipVolumeData: Option[Boolean], volumeDataZipFormat: Option[String]) POST /annotations/:id/acquireMutex controllers.AnnotationController.tryAcquiringAnnotationMutex(id: String) GET /annotations/:typ/:id/info controllers.AnnotationController.info(typ: String, id: String, timestamp: Option[Long]) -PATCH /annotations/:typ/:id/downsample controllers.AnnotationController.downsample(typ: String, id: String, tracingId: String) DELETE /annotations/:typ/:id controllers.AnnotationController.cancel(typ: String, id: String) POST /annotations/:typ/:id/merge/:mergedTyp/:mergedId controllers.AnnotationController.merge(typ: String, id: String, mergedTyp: String, mergedId: String) GET /annotations/:typ/:id/download controllers.AnnotationIOController.download(typ: String, id: String, skeletonVersion: Option[Long], volumeVersion: Option[Long], skipVolumeData: Option[Boolean], volumeDataZipFormat: Option[String]) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala index ba8aee37c6d..c0d9b8695d2 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala @@ -191,6 +191,11 @@ class RPCRequest(val id: Int, val url: String, wsClient: WSClient)(implicit ec: parseProtoResponse(performRequest)(companion) } + def postWithProtoResponse[T <: GeneratedMessage]()(companion: GeneratedMessageCompanion[T]): Fox[T] = { + request = request.withMethod("POST") + parseProtoResponse(performRequest)(companion) + } + private def performRequest: Fox[WSResponse] = { if (verbose) { logger.debug( diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 1240e66c9bf..5d472b3c1f6 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -146,10 +146,10 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } yield updated private def revertToVersion( - annotationId: String, - annotationWithTracings: AnnotationWithTracings, - revertAction: RevertToVersionAnnotationAction, - newVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = + annotationId: String, + annotationWithTracings: AnnotationWithTracings, + revertAction: RevertToVersionAnnotationAction, + newVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = // Note: works only after “ironing out” the update action groups // TODO: read old annotationProto, tracing, buckets, segment indeces for { @@ -612,4 +612,5 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } } + def duplicate(annotationId: String, version: Option[Long]): Fox[AnnotationProto] = ??? // TODO } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index 97315646e2c..2649ca295c4 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -100,4 +100,24 @@ class TSAnnotationController @Inject()( } } + def duplicate(annotationId: String, + version: Option[Long], + isFromTask: Option[Boolean], + minMag: Option[Int], + maxMag: Option[Int], + downsample: Option[Boolean], + editPosition: Option[String], + editRotation: Option[String], + boundingBox: Option[String]): Action[AnyContent] = + Action.async { implicit request => + log() { + logTime(slackNotificationService.noticeSlowRequest) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeAnnotation(annotationId)) { + for { + annotationProto <- annotationService.duplicate(annotationId, version) + } yield Ok(annotationProto.toByteArray).as(protobufMimeType) + } + } + } + } } diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index df237857e93..a94898a9f11 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -11,6 +11,7 @@ POST /annotation/:annotationId/update GET /annotation/:annotationId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionLog(annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]) GET /annotation/:annotationId/updateActionStatistics @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionStatistics(annotationId: String) GET /annotation/:annotationId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.newestVersion(annotationId: String) +POST /annotation/:annotationId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.duplicate(annotationId: String, version: Option[Long], isFromTask: Option[Boolean], minMag: Option[Int], maxMag: Option[Int], downsample: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) # Volume tracings POST /volume/save @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.save() From a007194e568ff1dfb2857193b97183813df7c0b3 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 24 Oct 2024 13:47:40 +0200 Subject: [PATCH 119/150] iterate on duplicate --- app/controllers/AnnotationController.scala | 5 ++- app/controllers/AnnotationIOController.scala | 3 +- app/models/annotation/AnnotationService.scala | 5 ++- .../WKRemoteTracingStoreClient.scala | 2 + .../annotation/TSAnnotationService.scala | 40 ++++++++++++++++++- .../controllers/TSAnnotationController.scala | 3 +- ...alableminds.webknossos.tracingstore.routes | 2 +- 7 files changed, 52 insertions(+), 8 deletions(-) diff --git a/app/controllers/AnnotationController.scala b/app/controllers/AnnotationController.scala index ff05b2b9cc9..9a915500a90 100755 --- a/app/controllers/AnnotationController.scala +++ b/app/controllers/AnnotationController.scala @@ -433,8 +433,10 @@ class AnnotationController @Inject()( datasetService.dataSourceFor(dataset).flatMap(_.toUsable).map(Some(_)) else Fox.successful(None) tracingStoreClient <- tracingStoreService.clientFor(dataset) + newAnnotationId = ObjectId.generate newAnnotationProto <- tracingStoreClient.duplicateAnnotation( annotation._id, + newAnnotationId, version = None, isFromTask = annotation._task.isDefined, editPosition = None, @@ -448,7 +450,8 @@ class AnnotationController @Inject()( newAnnotationLayers, AnnotationType.Explorational, None, - annotation.description) ?~> Messages("annotation.create.failed") + annotation.description, + newAnnotationId) ?~> Messages("annotation.create.failed") } yield clonedAnnotation def tryAcquiringAnnotationMutex(id: String): Action[AnyContent] = diff --git a/app/controllers/AnnotationIOController.scala b/app/controllers/AnnotationIOController.scala index 3215b05c3a7..08364247c6c 100755 --- a/app/controllers/AnnotationIOController.scala +++ b/app/controllers/AnnotationIOController.scala @@ -146,7 +146,8 @@ class AnnotationIOController @Inject()( mergedSkeletonLayers ::: mergedVolumeLayers, AnnotationType.Explorational, name, - description) + description, + ObjectId.generate) _ = analyticsService.track(UploadAnnotationEvent(request.identity, annotation)) } yield JsonOk( diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index 5c7cc8aa431..076bfce75ee 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -505,10 +505,11 @@ class AnnotationService @Inject()( annotationLayers: Seq[AnnotationLayer], annotationType: AnnotationType, name: Option[String], - description: String): Fox[Annotation] = + description: String, + newAnnotationId: ObjectId): Fox[Annotation] = for { teamId <- selectSuitableTeam(user, dataset) - annotation = Annotation(ObjectId.generate, + annotation = Annotation(newAnnotationId, dataset._id, None, teamId, diff --git a/app/models/annotation/WKRemoteTracingStoreClient.scala b/app/models/annotation/WKRemoteTracingStoreClient.scala index 017cc421d96..3ee30770dc1 100644 --- a/app/models/annotation/WKRemoteTracingStoreClient.scala +++ b/app/models/annotation/WKRemoteTracingStoreClient.scala @@ -91,6 +91,7 @@ class WKRemoteTracingStoreClient( } def duplicateAnnotation(annotationId: ObjectId, + newAnnotationId: ObjectId, version: Option[Long], isFromTask: Boolean, editPosition: Option[Vec3Int], @@ -101,6 +102,7 @@ class WKRemoteTracingStoreClient( logger.debug(s"Called to duplicate annotation $annotationId." + baseInfo) rpc(s"${tracingStore.url}/tracings/annotation/$annotationId/duplicate").withLongTimeout .addQueryString("token" -> RpcTokenHolder.webknossosToken) + .addQueryString("newAnnotationId" -> newAnnotationId.toString) .addQueryStringOptional("version", version.map(_.toString)) .addQueryStringOptional("editPosition", editPosition.map(_.toUriLiteral)) .addQueryStringOptional("editRotation", editRotation.map(_.toUriLiteral)) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 5d472b3c1f6..542e1cc27cf 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -5,7 +5,11 @@ import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox import com.scalableminds.util.tools.Fox.{box2Fox, option2Fox} -import com.scalableminds.webknossos.datastore.Annotation.{AnnotationLayerTypeProto, AnnotationProto} +import com.scalableminds.webknossos.datastore.Annotation.{ + AnnotationLayerProto, + AnnotationLayerTypeProto, + AnnotationProto +} import com.scalableminds.webknossos.datastore.EditableMappingInfo.EditableMappingInfo import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing @@ -612,5 +616,37 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } } - def duplicate(annotationId: String, version: Option[Long]): Fox[AnnotationProto] = ??? // TODO + // TODO duplicate v0 as well? (if current version is not v0) + def duplicate(annotationId: String, newAnnotationId: String, version: Option[Long])( + implicit ec: ExecutionContext, + tc: TokenContext): Fox[AnnotationProto] = + for { + current <- get(annotationId, version) + newLayers <- Fox.serialCombined(current.layers)(layer => duplicateLayer(annotationId, layer, version)) + duplicated = current.copy(layers = newLayers) + // TODO save duplicated + } yield duplicated + + private def duplicateLayer(annotationId: String, layer: AnnotationLayerProto, version: Option[Long])( + implicit ec: ExecutionContext, + tc: TokenContext): Fox[AnnotationLayerProto] = + for { + newTracingId <- layer.`type` match { + case AnnotationLayerTypeProto.volume => duplicateVolumeTracing(layer.tracingId, version) + case AnnotationLayerTypeProto.skeleton => duplicateSkeletonTracing(layer.tracingId, version) + case AnnotationLayerTypeProto.Unrecognized(num) => Fox.failure(f"unrecognized annotation layer type: $num") + } + } yield layer.copy(tracingId = newTracingId) + + private def duplicateVolumeTracing(tracingId: String, version: Option[Long])( + implicit ec: ExecutionContext): Fox[String] = { + val newTracingId = TracingId.generate + Fox.successful(newTracingId) + } + + private def duplicateSkeletonTracing(tracingId: String, version: Option[Long])( + implicit ec: ExecutionContext): Fox[String] = { + val newTracingId = TracingId.generate + Fox.successful(newTracingId) + } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index 2649ca295c4..6a7a232374a 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -101,6 +101,7 @@ class TSAnnotationController @Inject()( } def duplicate(annotationId: String, + newAnnotationId: String, version: Option[Long], isFromTask: Option[Boolean], minMag: Option[Int], @@ -114,7 +115,7 @@ class TSAnnotationController @Inject()( logTime(slackNotificationService.noticeSlowRequest) { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeAnnotation(annotationId)) { for { - annotationProto <- annotationService.duplicate(annotationId, version) + annotationProto <- annotationService.duplicate(annotationId, newAnnotationId, version) } yield Ok(annotationProto.toByteArray).as(protobufMimeType) } } diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index a94898a9f11..5edab1323f4 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -11,7 +11,7 @@ POST /annotation/:annotationId/update GET /annotation/:annotationId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionLog(annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]) GET /annotation/:annotationId/updateActionStatistics @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionStatistics(annotationId: String) GET /annotation/:annotationId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.newestVersion(annotationId: String) -POST /annotation/:annotationId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.duplicate(annotationId: String, version: Option[Long], isFromTask: Option[Boolean], minMag: Option[Int], maxMag: Option[Int], downsample: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) +POST /annotation/:annotationId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.duplicate(annotationId: String, newAnnotationId: String, version: Option[Long], isFromTask: Option[Boolean], minMag: Option[Int], maxMag: Option[Int], downsample: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) # Volume tracings POST /volume/save @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.save() From af0630a90d1822fae64e1237b456ed9cf7c37003 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 24 Oct 2024 14:07:40 +0200 Subject: [PATCH 120/150] iterate on duplicate --- .../annotation/TSAnnotationService.scala | 75 ++++++++++++++----- .../SkeletonTracingController.scala | 6 +- .../controllers/TSAnnotationController.scala | 15 +++- .../skeleton/SkeletonTracingService.scala | 13 ++-- ...alableminds.webknossos.tracingstore.routes | 2 +- 5 files changed, 78 insertions(+), 33 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 542e1cc27cf..18f4cc1df86 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -2,6 +2,7 @@ package com.scalableminds.webknossos.tracingstore.annotation import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.cache.AlfuCache +import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int} import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox import com.scalableminds.util.tools.Fox.{box2Fox, option2Fox} @@ -617,36 +618,74 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } // TODO duplicate v0 as well? (if current version is not v0) - def duplicate(annotationId: String, newAnnotationId: String, version: Option[Long])( - implicit ec: ExecutionContext, - tc: TokenContext): Fox[AnnotationProto] = + def duplicate( + annotationId: String, + newAnnotationId: String, + version: Option[Long], + isFromTask: Boolean, + editPosition: Option[Vec3Int], + editRotation: Option[Vec3Double], + boundingBox: Option[BoundingBox])(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationProto] = for { - current <- get(annotationId, version) - newLayers <- Fox.serialCombined(current.layers)(layer => duplicateLayer(annotationId, layer, version)) - duplicated = current.copy(layers = newLayers) - // TODO save duplicated - } yield duplicated - - private def duplicateLayer(annotationId: String, layer: AnnotationLayerProto, version: Option[Long])( - implicit ec: ExecutionContext, - tc: TokenContext): Fox[AnnotationLayerProto] = + currentAnnotation <- get(annotationId, version) + newLayers <- Fox.serialCombined(currentAnnotation.layers)( + layer => + duplicateLayer(annotationId, + layer, + currentAnnotation.version, + isFromTask, + editPosition, + editRotation, + boundingBox)) + duplicatedAnnotation = currentAnnotation.copy(layers = newLayers) + _ <- tracingDataStore.annotations.put(newAnnotationId, currentAnnotation.version, duplicatedAnnotation) + } yield duplicatedAnnotation + + private def duplicateLayer( + annotationId: String, + layer: AnnotationLayerProto, + version: Long, + isFromTask: Boolean, + editPosition: Option[Vec3Int], + editRotation: Option[Vec3Double], + boundingBox: Option[BoundingBox])(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationLayerProto] = for { newTracingId <- layer.`type` match { - case AnnotationLayerTypeProto.volume => duplicateVolumeTracing(layer.tracingId, version) - case AnnotationLayerTypeProto.skeleton => duplicateSkeletonTracing(layer.tracingId, version) + case AnnotationLayerTypeProto.volume => duplicateVolumeTracing(annotationId, layer.tracingId, version) + case AnnotationLayerTypeProto.skeleton => + duplicateSkeletonTracing(annotationId, + layer.tracingId, + version, + isFromTask, + editPosition, + editRotation, + boundingBox) case AnnotationLayerTypeProto.Unrecognized(num) => Fox.failure(f"unrecognized annotation layer type: $num") } } yield layer.copy(tracingId = newTracingId) - private def duplicateVolumeTracing(tracingId: String, version: Option[Long])( + private def duplicateVolumeTracing(annotationId: String, tracingId: String, version: Long)( implicit ec: ExecutionContext): Fox[String] = { val newTracingId = TracingId.generate Fox.successful(newTracingId) } - private def duplicateSkeletonTracing(tracingId: String, version: Option[Long])( - implicit ec: ExecutionContext): Fox[String] = { + private def duplicateSkeletonTracing( + annotationId: String, + tracingId: String, + version: Long, + isFromTask: Boolean, + editPosition: Option[Vec3Int], + editRotation: Option[Vec3Double], + boundingBox: Option[BoundingBox])(implicit ec: ExecutionContext, tc: TokenContext): Fox[String] = { val newTracingId = TracingId.generate - Fox.successful(newTracingId) + for { + skeleton <- findSkeleton(annotationId, tracingId, Some(version)) + adaptedSkeleton = skeletonTracingService + .adaptSkeletonForDuplicate(skeleton, isFromTask, editPosition, editRotation, boundingBox) + .withVersion(version) + _ <- tracingDataStore.skeletons.put(newTracingId, version, adaptedSkeleton) + } yield newTracingId } + } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala index f41279b4928..faa71d0711a 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala @@ -159,11 +159,7 @@ class SkeletonTracingController @Inject()(skeletonTracingService: SkeletonTracin editPositionParsed <- Fox.runOptional(editPosition)(Vec3Int.fromUriLiteral) editRotationParsed <- Fox.runOptional(editRotation)(Vec3Double.fromUriLiteral) boundingBoxParsed <- Fox.runOptional(boundingBox)(BoundingBox.fromLiteral) - newId <- skeletonTracingService.duplicate(tracing, - fromTask.getOrElse(false), - editPositionParsed, - editRotationParsed, - boundingBoxParsed) + newId = TracingId.generate } yield Ok(Json.toJson(newId)) } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index 6a7a232374a..69ee73b8150 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -1,6 +1,8 @@ package com.scalableminds.webknossos.tracingstore.controllers import com.google.inject.Inject +import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int} +import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.Annotation.AnnotationProto import com.scalableminds.webknossos.datastore.controllers.Controller import com.scalableminds.webknossos.datastore.services.UserAccessRequest @@ -103,7 +105,7 @@ class TSAnnotationController @Inject()( def duplicate(annotationId: String, newAnnotationId: String, version: Option[Long], - isFromTask: Option[Boolean], + isFromTask: Boolean, minMag: Option[Int], maxMag: Option[Int], downsample: Option[Boolean], @@ -115,7 +117,16 @@ class TSAnnotationController @Inject()( logTime(slackNotificationService.noticeSlowRequest) { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeAnnotation(annotationId)) { for { - annotationProto <- annotationService.duplicate(annotationId, newAnnotationId, version) + editPositionParsed <- Fox.runOptional(editPosition)(Vec3Int.fromUriLiteral) + editRotationParsed <- Fox.runOptional(editRotation)(Vec3Double.fromUriLiteral) + boundingBoxParsed <- Fox.runOptional(boundingBox)(BoundingBox.fromLiteral) + annotationProto <- annotationService.duplicate(annotationId, + newAnnotationId, + version, + isFromTask, + editPositionParsed, + editRotationParsed, + boundingBoxParsed) } yield Ok(annotationProto.toByteArray).as(protobufMimeType) } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala index dcbe1eb86c0..d25d2a4a79c 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala @@ -35,11 +35,11 @@ class SkeletonTracingService @Inject()( implicit val tracingCompanion: SkeletonTracing.type = SkeletonTracing - def duplicate(tracing: SkeletonTracing, - fromTask: Boolean, - editPosition: Option[Vec3Int], - editRotation: Option[Vec3Double], - boundingBox: Option[BoundingBox]): Fox[String] = { + def adaptSkeletonForDuplicate(tracing: SkeletonTracing, + fromTask: Boolean, + editPosition: Option[Vec3Int], + editRotation: Option[Vec3Double], + boundingBox: Option[BoundingBox]): SkeletonTracing = { val taskBoundingBox = if (fromTask) { tracing.boundingBox.map { bb => val newId = if (tracing.userBoundingBoxes.isEmpty) 1 else tracing.userBoundingBoxes.map(_.id).max + 1 @@ -57,8 +57,7 @@ class SkeletonTracingService @Inject()( version = 0 ) .addAllUserBoundingBoxes(taskBoundingBox) - val finalTracing = if (fromTask) newTracing.clearBoundingBox else newTracing - save(finalTracing, None, finalTracing.version) + if (fromTask) newTracing.clearBoundingBox else newTracing } def merge(tracings: Seq[SkeletonTracing], diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 5edab1323f4..f889ba4fde2 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -11,7 +11,7 @@ POST /annotation/:annotationId/update GET /annotation/:annotationId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionLog(annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]) GET /annotation/:annotationId/updateActionStatistics @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionStatistics(annotationId: String) GET /annotation/:annotationId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.newestVersion(annotationId: String) -POST /annotation/:annotationId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.duplicate(annotationId: String, newAnnotationId: String, version: Option[Long], isFromTask: Option[Boolean], minMag: Option[Int], maxMag: Option[Int], downsample: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) +POST /annotation/:annotationId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.duplicate(annotationId: String, newAnnotationId: String, version: Option[Long], isFromTask: Boolean, minMag: Option[Int], maxMag: Option[Int], downsample: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) # Volume tracings POST /volume/save @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.save() From 2fa24c6d4c45d62f776bce40850b80f8a1389813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Fri, 25 Oct 2024 14:44:11 +0200 Subject: [PATCH 121/150] do not add actionTracingId to updateTdCamera and revertToVersion update actions --- .../oxalis/model/reducers/save_reducer.ts | 24 ++++++++++--------- frontend/javascripts/oxalis/store.ts | 4 ++-- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts index 06fb2975175..73de1df5aca 100644 --- a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts +++ b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts @@ -196,17 +196,19 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState { export function addTracingIdToActions( actions: UpdateAction[], tracingId: string, -): UpdateActionWithTracingId[] { - return actions.map( - (innerAction) => - ({ - ...innerAction, - value: { - ...innerAction.value, - actionTracingId: tracingId, - }, - }) as UpdateActionWithTracingId, - ); +): Array { + return actions.map((action) => { + if (action.name === "updateTdCamera" || action.name === "revertToVersion") { + return action as UpdateAction; + } + return { + ...action, + value: { + ...action.value, + actionTracingId: tracingId, + }, + } as UpdateActionWithTracingId; + }); } export default SaveReducer; diff --git a/frontend/javascripts/oxalis/store.ts b/frontend/javascripts/oxalis/store.ts index 72c353756df..22585186ed6 100644 --- a/frontend/javascripts/oxalis/store.ts +++ b/frontend/javascripts/oxalis/store.ts @@ -50,7 +50,7 @@ import type { } from "oxalis/constants"; import type { BLEND_MODES, ControlModeEnum } from "oxalis/constants"; import type { Matrix4x4 } from "libs/mjs"; -import type { UpdateActionWithTracingId } from "oxalis/model/sagas/update_actions"; +import type { UpdateAction, UpdateActionWithTracingId } from "oxalis/model/sagas/update_actions"; import AnnotationReducer from "oxalis/model/reducers/annotation_reducer"; import DatasetReducer from "oxalis/model/reducers/dataset_reducer"; import type DiffableMap from "libs/diffable_map"; @@ -446,7 +446,7 @@ export type SaveQueueEntry = { version: number; timestamp: number; authorId: string; - actions: Array; + actions: Array; transactionId: string; transactionGroupCount: number; transactionGroupIndex: number; From 2f80f0f78e6ea98fcc43e3308af6e4f9f26935cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Fri, 25 Oct 2024 14:45:35 +0200 Subject: [PATCH 122/150] some frontend spec fixes (far more are still needed xD) --- .../javascripts/test/sagas/save_saga.spec.ts | 18 +++--------------- .../test/sagas/skeletontracing_saga.spec.ts | 3 ++- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/frontend/javascripts/test/sagas/save_saga.spec.ts b/frontend/javascripts/test/sagas/save_saga.spec.ts index 8036007eb96..a7881fb031d 100644 --- a/frontend/javascripts/test/sagas/save_saga.spec.ts +++ b/frontend/javascripts/test/sagas/save_saga.spec.ts @@ -301,11 +301,7 @@ test("SaveSaga should remove the correct update actions", (t) => { ]); saga.next(annotationId); saga.next(TRACINGSTORE_URL); - expectValueDeepEqual( - t, - saga.next(), - put(SaveActions.setVersionNumberAction(3, TRACING_TYPE, tracingId)), - ); + expectValueDeepEqual(t, saga.next(), put(SaveActions.setVersionNumberAction(3))); expectValueDeepEqual(t, saga.next(), put(SaveActions.setLastSaveTimestampAction())); expectValueDeepEqual(t, saga.next(), put(SaveActions.shiftSaveQueueAction(2))); }); @@ -331,11 +327,7 @@ test("SaveSaga should set the correct version numbers", (t) => { ]); saga.next(annotationId); saga.next(TRACINGSTORE_URL); - expectValueDeepEqual( - t, - saga.next(), - put(SaveActions.setVersionNumberAction(LAST_VERSION + 3, TRACING_TYPE, tracingId)), - ); + expectValueDeepEqual(t, saga.next(), put(SaveActions.setVersionNumberAction(LAST_VERSION + 3))); expectValueDeepEqual(t, saga.next(), put(SaveActions.setLastSaveTimestampAction())); expectValueDeepEqual(t, saga.next(), put(SaveActions.shiftSaveQueueAction(3))); }); @@ -362,11 +354,7 @@ test("SaveSaga should set the correct version numbers if the save queue was comp saga.next(annotationId); saga.next(TRACINGSTORE_URL); // two of the updateTracing update actions are removed by compactSaveQueue - expectValueDeepEqual( - t, - saga.next(), - put(SaveActions.setVersionNumberAction(LAST_VERSION + 1, TRACING_TYPE, tracingId)), - ); + expectValueDeepEqual(t, saga.next(), put(SaveActions.setVersionNumberAction(LAST_VERSION + 1))); expectValueDeepEqual(t, saga.next(), put(SaveActions.setLastSaveTimestampAction())); expectValueDeepEqual(t, saga.next(), put(SaveActions.shiftSaveQueueAction(3))); }); diff --git a/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts b/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts index 5d8e449b456..1e1011bba7c 100644 --- a/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts +++ b/frontend/javascripts/test/sagas/skeletontracing_saga.spec.ts @@ -79,6 +79,8 @@ function testDiffing( ); } +// TODOM +// biome-ignore lint/correctness/noUnusedVariables: function compactSaveQueueWithUpdateActions( queue: Array, tracing: SkeletonTracing, @@ -90,7 +92,6 @@ function compactSaveQueueWithUpdateActions( // filling the save queue). one could probably combine compactUpdateActions and // createSaveQueueFromUpdateActions to have a createCompactedSaveQueueFromUpdateActions // helper function and use that in this spec. - // @ts-expect-error queue.map((batch) => ({ ...batch, actions: compactUpdateActions(batch.actions, tracing) })), ); } From 52076c13f141a2c968c63fc6db35ddb89dfbdddb Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 28 Oct 2024 10:04:18 +0100 Subject: [PATCH 123/150] iterate on duplicate (volumes, updates, buckets) --- .../annotation/TSAnnotationService.scala | 71 +++++++++++++++--- .../SkeletonTracingController.scala | 23 ------ .../controllers/TSAnnotationController.scala | 7 +- .../controllers/VolumeTracingController.scala | 48 ------------- .../skeleton/SkeletonTracingService.scala | 5 +- .../volume/VolumeTracingService.scala | 72 +++++++++---------- ...alableminds.webknossos.tracingstore.routes | 2 - 7 files changed, 103 insertions(+), 125 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 18f4cc1df86..e5e2ac90f84 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -31,6 +31,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.skeleton.updating.{ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ ApplyableVolumeUpdateAction, BucketMutatingVolumeUpdateAction, + MagRestrictions, UpdateMappingNameVolumeAction, VolumeTracingService, VolumeUpdateAction @@ -625,7 +626,8 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss isFromTask: Boolean, editPosition: Option[Vec3Int], editRotation: Option[Vec3Double], - boundingBox: Option[BoundingBox])(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationProto] = + boundingBox: Option[BoundingBox], + magRestrictions: MagRestrictions)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationProto] = for { currentAnnotation <- get(annotationId, version) newLayers <- Fox.serialCombined(currentAnnotation.layers)( @@ -636,11 +638,25 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss isFromTask, editPosition, editRotation, - boundingBox)) + boundingBox, + magRestrictions)) + _ <- duplicateUpdates(annotationId, newAnnotationId) duplicatedAnnotation = currentAnnotation.copy(layers = newLayers) _ <- tracingDataStore.annotations.put(newAnnotationId, currentAnnotation.version, duplicatedAnnotation) } yield duplicatedAnnotation + private def duplicateUpdates(annotationId: String, newAnnotationId: String)( + implicit ec: ExecutionContext): Fox[Unit] = + // TODO perf: batch or use fossildb duplicate api + for { + updatesAsBytes: Seq[(Long, Array[Byte])] <- tracingDataStore.annotationUpdates + .getMultipleVersionsAsVersionValueTuple(annotationId) + _ <- Fox.serialCombined(updatesAsBytes) { + case (version, updateBytes) => + tracingDataStore.annotationUpdates.put(newAnnotationId, version, updateBytes) + } + } yield () + private def duplicateLayer( annotationId: String, layer: AnnotationLayerProto, @@ -648,10 +664,19 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss isFromTask: Boolean, editPosition: Option[Vec3Int], editRotation: Option[Vec3Double], - boundingBox: Option[BoundingBox])(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationLayerProto] = + boundingBox: Option[BoundingBox], + magRestrictions: MagRestrictions)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationLayerProto] = for { newTracingId <- layer.`type` match { - case AnnotationLayerTypeProto.volume => duplicateVolumeTracing(annotationId, layer.tracingId, version) + case AnnotationLayerTypeProto.volume => + duplicateVolumeTracing(annotationId, + layer.tracingId, + version, + isFromTask, + boundingBox, + magRestrictions, + editPosition, + editRotation) case AnnotationLayerTypeProto.skeleton => duplicateSkeletonTracing(annotationId, layer.tracingId, @@ -664,10 +689,33 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } } yield layer.copy(tracingId = newTracingId) - private def duplicateVolumeTracing(annotationId: String, tracingId: String, version: Long)( - implicit ec: ExecutionContext): Fox[String] = { + private def duplicateVolumeTracing( + annotationId: String, + sourceTracingId: String, + version: Long, + isFromTask: Boolean, + boundingBox: Option[BoundingBox], + magRestrictions: MagRestrictions, + editPosition: Option[Vec3Int], + editRotation: Option[Vec3Double])(implicit ec: ExecutionContext, tc: TokenContext): Fox[String] = { val newTracingId = TracingId.generate - Fox.successful(newTracingId) + for { + sourceTracing <- findVolume(annotationId, sourceTracingId, Some(version)) + newTracing <- volumeTracingService.adaptVolumeForDuplicate(sourceTracingId, + newTracingId, + sourceTracing, + isFromTask, + boundingBox, + magRestrictions, + editPosition, + editRotation, + version) + _ <- tracingDataStore.volumes.put(newTracingId, version, newTracing) + _ <- volumeTracingService.duplicateVolumeData(sourceTracingId, sourceTracing, newTracingId, newTracing) + /*_ <- Fox.runIf(adaptedTracing.getHasEditableMapping)( + editableMappingService.duplicate(tracingId, newTracingId, version = None, remoteFallbackLayerOpt))*/ + // TODO downsample? + } yield newTracingId } private def duplicateSkeletonTracing( @@ -681,9 +729,12 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss val newTracingId = TracingId.generate for { skeleton <- findSkeleton(annotationId, tracingId, Some(version)) - adaptedSkeleton = skeletonTracingService - .adaptSkeletonForDuplicate(skeleton, isFromTask, editPosition, editRotation, boundingBox) - .withVersion(version) + adaptedSkeleton = skeletonTracingService.adaptSkeletonForDuplicate(skeleton, + isFromTask, + editPosition, + editRotation, + boundingBox, + version) _ <- tracingDataStore.skeletons.put(newTracingId, version, adaptedSkeleton) } yield newTracingId } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala index faa71d0711a..bcfe15941fd 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala @@ -1,7 +1,6 @@ package com.scalableminds.webknossos.tracingstore.controllers import com.google.inject.Inject -import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int} import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.SkeletonTracing.{SkeletonTracing, SkeletonTracingOpt, SkeletonTracings} import com.scalableminds.webknossos.datastore.services.UserAccessRequest @@ -143,26 +142,4 @@ class SkeletonTracingController @Inject()(skeletonTracingService: SkeletonTracin } } - def duplicate(tracingId: String, - version: Option[Long], - fromTask: Option[Boolean], - editPosition: Option[String], - editRotation: Option[String], - boundingBox: Option[String]): Action[AnyContent] = - Action.async { implicit request => - log() { - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { - for { - annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- annotationService.findSkeleton(annotationId, tracingId, version, applyUpdates = true) ?~> Messages( - "tracing.notFound") - editPositionParsed <- Fox.runOptional(editPosition)(Vec3Int.fromUriLiteral) - editRotationParsed <- Fox.runOptional(editRotation)(Vec3Double.fromUriLiteral) - boundingBoxParsed <- Fox.runOptional(boundingBox)(BoundingBox.fromLiteral) - newId = TracingId.generate - } yield Ok(Json.toJson(newId)) - } - } - } - } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index 69ee73b8150..ea689011306 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -14,6 +14,7 @@ import com.scalableminds.webknossos.tracingstore.annotation.{ UpdateActionGroup } import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService +import com.scalableminds.webknossos.tracingstore.tracings.volume.MagRestrictions import play.api.libs.json.Json import play.api.mvc.{Action, AnyContent, PlayBodyParsers} @@ -108,7 +109,7 @@ class TSAnnotationController @Inject()( isFromTask: Boolean, minMag: Option[Int], maxMag: Option[Int], - downsample: Option[Boolean], + downsample: Option[Boolean], // TODO remove or implement editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]): Action[AnyContent] = @@ -120,13 +121,15 @@ class TSAnnotationController @Inject()( editPositionParsed <- Fox.runOptional(editPosition)(Vec3Int.fromUriLiteral) editRotationParsed <- Fox.runOptional(editRotation)(Vec3Double.fromUriLiteral) boundingBoxParsed <- Fox.runOptional(boundingBox)(BoundingBox.fromLiteral) + magRestrictions = MagRestrictions(minMag, maxMag) annotationProto <- annotationService.duplicate(annotationId, newAnnotationId, version, isFromTask, editPositionParsed, editRotationParsed, - boundingBoxParsed) + boundingBoxParsed, + magRestrictions) } yield Ok(annotationProto.toByteArray).as(protobufMimeType) } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 0b0305f627c..a56a441fba6 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -277,54 +277,6 @@ class VolumeTracingController @Inject()( private def formatMissingBucketList(indices: List[Int]): String = "[" + indices.mkString(", ") + "]" - def duplicate(tracingId: String, - fromTask: Option[Boolean], - minMag: Option[Int], - maxMag: Option[Int], - downsample: Option[Boolean], - editPosition: Option[String], - editRotation: Option[String], - boundingBox: Option[String]): Action[AnyContent] = Action.async { implicit request => - log() { - logTime(slackNotificationService.noticeSlowRequest) { - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { - for { - annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) - tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") - _ = logger.info(s"Duplicating volume tracing $tracingId...") - datasetBoundingBox = request.body.asJson.flatMap(_.validateOpt[BoundingBox].asOpt.flatten) - magRestrictions = MagRestrictions(minMag, maxMag) - editPositionParsed <- Fox.runOptional(editPosition)(Vec3Int.fromUriLiteral) - editRotationParsed <- Fox.runOptional(editRotation)(Vec3Double.fromUriLiteral) - boundingBoxParsed <- Fox.runOptional(boundingBox)(BoundingBox.fromLiteral) - remoteFallbackLayerOpt <- Fox.runIf(tracing.getHasEditableMapping)( - volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId)) - newTracingId = TracingId.generate - // TODO - /*_ <- Fox.runIf(tracing.getHasEditableMapping)( - editableMappingService.duplicate(tracingId, newTracingId, version = None, remoteFallbackLayerOpt))*/ - // TODO actionTracingIds + addLayer tracing ids need to be remapped (as they need to be globally unique) - (newId, newTracing) <- volumeTracingService.duplicate( - annotationId, - tracingId, - newTracingId, - tracing, - fromTask.getOrElse(false), - datasetBoundingBox, - magRestrictions, - editPositionParsed, - editRotationParsed, - boundingBoxParsed, - mappingName = None - ) - _ <- Fox.runIfOptionTrue(downsample)( - volumeTracingService.downsample(annotationId, newId, tracingId, newTracing)) - } yield Ok(Json.toJson(newId)) - } - } - } - } - def importVolumeData(tracingId: String): Action[MultipartFormData[TemporaryFile]] = Action.async(parse.multipartFormData) { implicit request => log() { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala index d25d2a4a79c..700c13c41c5 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala @@ -39,7 +39,8 @@ class SkeletonTracingService @Inject()( fromTask: Boolean, editPosition: Option[Vec3Int], editRotation: Option[Vec3Double], - boundingBox: Option[BoundingBox]): SkeletonTracing = { + boundingBox: Option[BoundingBox], + newVersion: Long): SkeletonTracing = { val taskBoundingBox = if (fromTask) { tracing.boundingBox.map { bb => val newId = if (tracing.userBoundingBoxes.isEmpty) 1 else tracing.userBoundingBoxes.map(_.id).max + 1 @@ -54,7 +55,7 @@ class SkeletonTracingService @Inject()( editPosition = editPosition.map(vec3IntToProto).getOrElse(tracing.editPosition), editRotation = editRotation.map(vec3DoubleToProto).getOrElse(tracing.editRotation), boundingBox = boundingBoxOptToProto(boundingBox).orElse(tracing.boundingBox), - version = 0 + version = newVersion ) .addAllUserBoundingBoxes(taskBoundingBox) if (fromTask) newTracing.clearBoundingBox else newTracing diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 1f0d68a805d..291fd749f39 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -486,38 +486,34 @@ class VolumeTracingService @Inject()( data <- binaryDataService.handleDataRequests(requests) } yield data - def duplicate(annotationId: String, - tracingId: String, - newTracingId: String, - sourceTracing: VolumeTracing, - fromTask: Boolean, - datasetBoundingBox: Option[BoundingBox], - magRestrictions: MagRestrictions, - editPosition: Option[Vec3Int], - editRotation: Option[Vec3Double], - boundingBox: Option[BoundingBox], - mappingName: Option[String])(implicit tc: TokenContext): Fox[(String, VolumeTracing)] = { - val tracingWithBB = addBoundingBoxFromTaskIfRequired(sourceTracing, fromTask, datasetBoundingBox) + def adaptVolumeForDuplicate(sourceTracingId: String, + newTracingId: String, + sourceTracing: VolumeTracing, + isFromTask: Boolean, + boundingBox: Option[BoundingBox], + magRestrictions: MagRestrictions, + editPosition: Option[Vec3Int], + editRotation: Option[Vec3Double], + newVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[VolumeTracing] = { + val tracingWithBB = addBoundingBoxFromTaskIfRequired(sourceTracing, isFromTask, boundingBox) val tracingWithMagRestrictions = restrictMagList(tracingWithBB, magRestrictions) for { - fallbackLayer <- getFallbackLayer(tracingId, sourceTracing) + fallbackLayer <- getFallbackLayer(sourceTracingId, sourceTracing) hasSegmentIndex <- VolumeSegmentIndexService.canHaveSegmentIndex(remoteDatastoreClient, fallbackLayer) newTracing = tracingWithMagRestrictions.copy( createdTimestamp = System.currentTimeMillis(), - editPosition = editPosition.map(vec3IntToProto).getOrElse(tracingWithMagRestrictions.editPosition), - editRotation = editRotation.map(vec3DoubleToProto).getOrElse(tracingWithMagRestrictions.editRotation), - boundingBox = boundingBoxOptToProto(boundingBox).getOrElse(tracingWithMagRestrictions.boundingBox), - mappingName = mappingName.orElse( + editPosition = editPosition.map(vec3IntToProto).getOrElse(sourceTracing.editPosition), + editRotation = editRotation.map(vec3DoubleToProto).getOrElse(sourceTracing.editRotation), + boundingBox = boundingBoxOptToProto(boundingBox).getOrElse(sourceTracing.boundingBox), + mappingName = if (sourceTracing.getHasEditableMapping) Some(newTracingId) - else tracingWithMagRestrictions.mappingName), - version = 0, + else sourceTracing.mappingName, + version = newVersion, // Adding segment index on duplication if the volume tracing allows it. This will be used in duplicateData hasSegmentIndex = Some(hasSegmentIndex) ) _ <- bool2Fox(newTracing.mags.nonEmpty) ?~> "magRestrictions.tooTight" - newId <- save(newTracing, Some(newTracingId), newTracing.version) - _ <- duplicateData(annotationId, tracingId, sourceTracing, newId, newTracing) - } yield (newId, newTracing) + } yield newTracing } @SuppressWarnings(Array("OptionGet")) //We suppress this warning because we check the option beforehand @@ -536,21 +532,21 @@ class VolumeTracingService @Inject()( .withBoundingBox(datasetBoundingBox.get) } else tracing - private def duplicateData(annotationId: String, - sourceId: String, - sourceTracing: VolumeTracing, - destinationId: String, - destinationTracing: VolumeTracing)(implicit tc: TokenContext): Fox[Unit] = + def duplicateVolumeData(sourceTracingId: String, + sourceTracing: VolumeTracing, + newTracingId: String, + newTracing: VolumeTracing)(implicit tc: TokenContext): Fox[Unit] = for { - isTemporaryTracing <- isTemporaryTracing(sourceId) - sourceDataLayer = volumeTracingLayer(sourceId, sourceTracing, isTemporaryTracing) - buckets: Iterator[(BucketPosition, Array[Byte])] = sourceDataLayer.bucketProvider.bucketStream() - destinationDataLayer = volumeTracingLayer(destinationId, destinationTracing) - fallbackLayer <- getFallbackLayer(sourceId, sourceTracing) + isTemporaryTracing <- isTemporaryTracing(sourceTracingId) + sourceDataLayer = volumeTracingLayer(sourceTracingId, sourceTracing, isTemporaryTracing) + buckets: Iterator[(BucketPosition, Array[Byte])] = sourceDataLayer.bucketProvider.bucketStream( + Some(newTracing.version)) + destinationDataLayer = volumeTracingLayer(newTracingId, newTracing) + fallbackLayer <- getFallbackLayer(sourceTracingId, sourceTracing) segmentIndexBuffer = new VolumeSegmentIndexBuffer( - destinationId, + newTracingId, volumeSegmentIndexClient, - destinationTracing.version, + newTracing.version, remoteDatastoreClient, fallbackLayer, AdditionalAxis.fromProtosAsOpt(sourceTracing.additionalAxes), @@ -559,10 +555,10 @@ class VolumeTracingService @Inject()( mappingName <- selectMappingName(sourceTracing) _ <- Fox.serialCombined(buckets) { case (bucketPosition, bucketData) => - if (destinationTracing.mags.contains(vec3IntToProto(bucketPosition.mag))) { + if (newTracing.mags.contains(vec3IntToProto(bucketPosition.mag))) { for { - _ <- saveBucket(destinationDataLayer, bucketPosition, bucketData, destinationTracing.version) - _ <- Fox.runIfOptionTrue(destinationTracing.hasSegmentIndex)( + _ <- saveBucket(destinationDataLayer, bucketPosition, bucketData, newTracing.version) + _ <- Fox.runIfOptionTrue(newTracing.hasSegmentIndex)( updateSegmentIndex( segmentIndexBuffer, bucketPosition, @@ -570,7 +566,7 @@ class VolumeTracingService @Inject()( Empty, sourceTracing.elementClass, mappingName, - editableMappingTracingId(sourceTracing, sourceId) + editableMappingTracingId(sourceTracing, sourceTracingId) )) } yield () } else Fox.successful(()) diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index f889ba4fde2..5caca298037 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -20,7 +20,6 @@ POST /volume/:tracingId/initialDataMultiple GET /volume/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.get(tracingId: String, version: Option[Long]) GET /volume/:tracingId/allDataZip @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.allDataZip(tracingId: String, volumeDataZipFormat: String, version: Option[Long], voxelSize: Option[String], voxelSizeUnit: Option[String]) POST /volume/:tracingId/data @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.data(tracingId: String) -POST /volume/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.duplicate(tracingId: String, fromTask: Option[Boolean], minMag: Option[Int], maxMag: Option[Int], downsample: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) POST /volume/:tracingId/adHocMesh @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.requestAdHocMesh(tracingId: String) POST /volume/:tracingId/fullMesh.stl @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.loadFullMeshStl(tracingId: String) POST /volume/:tracingId/segmentIndex/:segmentId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentIndex(tracingId: String, segmentId: Long) @@ -74,4 +73,3 @@ POST /skeleton/mergedFromContents POST /skeleton/mergedFromIds @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.mergedFromIds(persist: Boolean) GET /skeleton/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.get(tracingId: String, version: Option[Long]) POST /skeleton/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.getMultiple -POST /skeleton/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.duplicate(tracingId: String, version: Option[Long], fromTask: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) From ce147181eb4d8d434ddffea6d8e3b3f145d45265 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 28 Oct 2024 10:17:32 +0100 Subject: [PATCH 124/150] duplicate editable mapping --- .../annotation/TSAnnotationService.scala | 22 ++++++++--- .../EditableMappingController.scala | 10 ++--- .../EditableMappingLayer.scala | 2 +- .../EditableMappingService.scala | 37 +++---------------- 4 files changed, 29 insertions(+), 42 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index e5e2ac90f84..7b4370d3855 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -248,7 +248,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss reportChangesToWk) ?~> "applyUpdates.failed" } yield updated - def getEditableMappingInfo(annotationId: String, tracingId: String, version: Option[Long] = None)( + def findEditableMappingInfo(annotationId: String, tracingId: String, version: Option[Long] = None)( implicit ec: ExecutionContext, tc: TokenContext): Fox[EditableMappingInfo] = for { @@ -546,7 +546,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss tc: TokenContext): Fox[Option[String]] = if (tracing.getHasEditableMapping) for { - editableMappingInfo <- getEditableMappingInfo(annotationId, tracingId) + editableMappingInfo <- findEditableMappingInfo(annotationId, tracingId) } yield Some(editableMappingInfo.baseMappingName) else Fox.successful(tracing.mappingName) @@ -711,13 +711,25 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss editRotation, version) _ <- tracingDataStore.volumes.put(newTracingId, version, newTracing) - _ <- volumeTracingService.duplicateVolumeData(sourceTracingId, sourceTracing, newTracingId, newTracing) - /*_ <- Fox.runIf(adaptedTracing.getHasEditableMapping)( - editableMappingService.duplicate(tracingId, newTracingId, version = None, remoteFallbackLayerOpt))*/ + _ <- Fox.runIf(!newTracing.getHasEditableMapping)( + volumeTracingService.duplicateVolumeData(sourceTracingId, sourceTracing, newTracingId, newTracing)) + _ <- Fox.runIf(newTracing.getHasEditableMapping)( + duplicateEditableMapping(annotationId, sourceTracingId, newTracingId, version)) // TODO downsample? } yield newTracingId } + private def duplicateEditableMapping(annotationId: String, + sourceTracingId: String, + newTracingId: String, + version: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Unit] = + for { + editableMappingInfo <- findEditableMappingInfo(annotationId, sourceTracingId, Some(version)) + _ <- tracingDataStore.editableMappingsInfo.put(newTracingId, version, toProtoBytes(editableMappingInfo)) + _ <- editableMappingService.duplicateSegmentToAgglomerate(sourceTracingId, newTracingId, version) + _ <- editableMappingService.duplicateAgglomerateToGraph(sourceTracingId, newTracingId, version) + } yield () + private def duplicateSkeletonTracing( annotationId: String, tracingId: String, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala index 8e5cc234a5e..5abf9014d24 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala @@ -113,7 +113,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- annotationService.findVolume(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) - editableMappingInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId, version) + editableMappingInfo <- annotationService.findEditableMappingInfo(annotationId, tracingId, version) infoJson = editableMappingService.infoJson(tracingId = tracingId, editableMappingInfo = editableMappingInfo) } yield Ok(infoJson) } @@ -151,7 +151,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer tracing <- annotationService.findVolume(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - editableMappingInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId, version = None) + editableMappingInfo <- annotationService.findEditableMappingInfo(annotationId, tracingId, version = None) relevantMapping: Map[Long, Long] <- editableMappingService.generateCombinedMappingForSegmentIds( request.body.items.toSet, editableMappingInfo, @@ -173,7 +173,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer tracing <- annotationService.findVolume(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - editableMappingInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId) + editableMappingInfo <- annotationService.findEditableMappingInfo(annotationId, tracingId) edges <- editableMappingService.agglomerateGraphMinCut(tracingId, tracing.version, editableMappingInfo, @@ -193,7 +193,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer tracing <- annotationService.findVolume(annotationId, tracingId) _ <- editableMappingService.assertTracingHasEditableMapping(tracing) remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) - editableMappingInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId) + editableMappingInfo <- annotationService.findEditableMappingInfo(annotationId, tracingId) (segmentId, edges) <- editableMappingService.agglomerateGraphNeighbors(tracingId, editableMappingInfo, tracing.version, @@ -211,7 +211,7 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) tracing <- annotationService.findVolume(annotationId, tracingId) _ <- bool2Fox(tracing.getHasEditableMapping) ?~> "Cannot query agglomerate skeleton for volume annotation" - editableMappingInfo <- annotationService.getEditableMappingInfo(annotationId, tracingId) + editableMappingInfo <- annotationService.findEditableMappingInfo(annotationId, tracingId) remoteFallbackLayer <- volumeTracingService.remoteFallbackLayerFromVolumeTracing(tracing, tracingId) agglomerateSkeletonBytes <- editableMappingService.getAgglomerateSkeletonWithFallback(tracingId, tracing.version, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala index 8b1561ac8ad..1f320a38d59 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala @@ -36,7 +36,7 @@ class EditableMappingBucketProvider(layer: EditableMappingLayer) extends BucketP remoteFallbackLayer <- layer.editableMappingService .remoteFallbackLayerFromVolumeTracing(layer.tracing, layer.tracingId) // called here to ensure updates are applied - editableMappingInfo <- layer.annotationService.getEditableMappingInfo(layer.annotationId, + editableMappingInfo <- layer.annotationService.findEditableMappingInfo(layer.annotationId, tracingId, Some(layer.version))(ec, layer.tokenContext) dataRequest: WebknossosDataRequest = WebknossosDataRequest( diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index dcc8c012cc6..66dd0e32eb7 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -135,60 +135,35 @@ class EditableMappingService @Inject()( } yield newEditableMappingInfo } - /* TODO - def duplicate(sourceTracingId: String, - newTracingId: String, - version: Option[Long], - remoteFallbackLayerBox: Box[RemoteFallbackLayer])(implicit tc: TokenContext): Fox[Unit] = - for { - remoteFallbackLayer <- remoteFallbackLayerBox ?~> "duplicate on editable mapping without remote fallback layer" - (duplicatedInfo, newVersion) <- getInfoAndActualVersion(sourceTracingId, version, remoteFallbackLayer) - _ <- tracingDataStore.editableMappingsInfo.put(newTracingId, newVersion, toProtoBytes(duplicatedInfo)) - _ <- duplicateSegmentToAgglomerate(sourceTracingId, newTracingId, newVersion) - _ <- duplicateAgglomerateToGraph(sourceTracingId, newTracingId, newVersion) - updateActionsWithVersions <- getUpdateActionsWithVersions(sourceTracingId, newVersion, 0L) - _ <- Fox.serialCombined(updateActionsWithVersions) { updateActionsWithVersion: (Long, List[UpdateAction]) => - tracingDataStore.editableMappingUpdates.put(newTracingId, - updateActionsWithVersion._1, - updateActionsWithVersion._2) - } - } yield () - - - private def duplicateSegmentToAgglomerate(sourceTracingId: String, newId: String, newVersion: Long): Fox[Unit] = { + def duplicateSegmentToAgglomerate(sourceTracingId: String, newId: String, version: Long): Fox[Unit] = { val iterator = new VersionedFossilDbIterator(sourceTracingId, tracingDataStore.editableMappingsSegmentToAgglomerate, - Some(newVersion)) + Some(version)) for { _ <- Fox.combined(iterator.map { keyValuePair => for { chunkId <- chunkIdFromSegmentToAgglomerateKey(keyValuePair.key).toFox newKey = segmentToAgglomerateKey(newId, chunkId) - _ <- tracingDataStore.editableMappingsSegmentToAgglomerate.put(newKey, - version = newVersion, - keyValuePair.value) + _ <- tracingDataStore.editableMappingsSegmentToAgglomerate.put(newKey, version = version, keyValuePair.value) } yield () }.toList) } yield () } - private def duplicateAgglomerateToGraph(sourceTracingId: String, newId: String, newVersion: Long): Fox[Unit] = { + def duplicateAgglomerateToGraph(sourceTracingId: String, newId: String, version: Long): Fox[Unit] = { val iterator = - new VersionedFossilDbIterator(sourceTracingId, - tracingDataStore.editableMappingsAgglomerateToGraph, - Some(newVersion)) + new VersionedFossilDbIterator(sourceTracingId, tracingDataStore.editableMappingsAgglomerateToGraph, Some(version)) for { _ <- Fox.combined(iterator.map { keyValuePair => for { agglomerateId <- agglomerateIdFromAgglomerateGraphKey(keyValuePair.key).toFox newKey = agglomerateGraphKey(newId, agglomerateId) - _ <- tracingDataStore.editableMappingsAgglomerateToGraph.put(newKey, version = newVersion, keyValuePair.value) + _ <- tracingDataStore.editableMappingsAgglomerateToGraph.put(newKey, version = version, keyValuePair.value) } yield () }.toList) } yield () } - */ def assertTracingHasEditableMapping(tracing: VolumeTracing)(implicit ec: ExecutionContext): Fox[Unit] = bool2Fox(tracing.getHasEditableMapping) ?~> "annotation.volume.noEditableMapping" From ba476478555d3d3aa4e3e740f1fa285d4b0d8841 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 28 Oct 2024 10:24:49 +0100 Subject: [PATCH 125/150] do not report layer changes to postgres when loading an older version --- .../tracingstore/annotation/TSAnnotationService.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 7b4370d3855..3d96cbe2ede 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -458,7 +458,9 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss _ <- updatedWithNewVerson.flushBufferedUpdates() _ <- flushUpdatedTracings(updatedWithNewVerson) _ <- flushAnnotationInfo(annotationId, updatedWithNewVerson) - _ <- remoteWebknossosClient.updateAnnotation(annotationId, updatedWithNewVerson.annotation) // TODO perf: skip if annotation is identical + _ <- Fox.runIf(reportChangesToWk)(remoteWebknossosClient.updateAnnotation( + annotationId, + updatedWithNewVerson.annotation)) // TODO perf: skip if annotation is identical } yield updatedWithNewVerson } } @@ -492,10 +494,9 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss * hence the emptyFallbck annotation.version) */ for { - newestUpdateVersion <- tracingDataStore.annotationUpdates.getVersion( - annotationId, - mayBeEmpty = Some(true), - emptyFallback = Some(0L)) // TODO in case of empty, look in annotation table, take version from there + newestUpdateVersion <- tracingDataStore.annotationUpdates.getVersion(annotationId, + mayBeEmpty = Some(true), + emptyFallback = Some(0L)) } yield { targetVersionOpt match { case None => newestUpdateVersion From b3f9a2c659a137f1fb3ee0fa274e942525c68b4a Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 28 Oct 2024 11:07:02 +0100 Subject: [PATCH 126/150] some cleanup --- app/models/annotation/AnnotationService.scala | 10 +-------- .../WKRemoteTracingStoreClient.scala | 12 +--------- .../annotation/AnnotationReversion.scala | 4 +--- .../annotation/AnnotationUpdateActions.scala | 5 ++++- .../annotation/AnnotationWithTracings.scala | 3 ++- .../annotation/TSAnnotationService.scala | 22 +++++++++---------- .../controllers/TSAnnotationController.scala | 3 --- ...alableminds.webknossos.tracingstore.routes | 2 +- 8 files changed, 21 insertions(+), 40 deletions(-) diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index 076bfce75ee..fc2c4bb1c42 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -305,15 +305,7 @@ class AnnotationService @Inject()( def downsampleAnnotation(annotation: Annotation, volumeAnnotationLayer: AnnotationLayer)( implicit ctx: DBAccessContext): Fox[Unit] = - for { - dataset <- datasetDAO.findOne(annotation._dataset) ?~> "dataset.notFoundForAnnotation" - _ <- bool2Fox(volumeAnnotationLayer.typ == AnnotationLayerType.Volume) ?~> "annotation.downsample.volumeOnly" - rpcClient <- tracingStoreService.clientFor(dataset) - newVolumeTracingId <- rpcClient.duplicateVolumeTracing(volumeAnnotationLayer.tracingId, downsample = true) - _ = logger.info( - s"Replacing volume tracing ${volumeAnnotationLayer.tracingId} by downsampled copy $newVolumeTracingId for annotation ${annotation._id}.") - _ <- annotationLayersDAO.replaceTracingId(annotation._id, volumeAnnotationLayer.tracingId, newVolumeTracingId) - } yield () + ??? // TODO: remove feature or implement as update action // WARNING: needs to be repeatable, might be called multiple times for an annotation def finish(annotation: Annotation, user: User, restrictions: AnnotationRestrictions)( diff --git a/app/models/annotation/WKRemoteTracingStoreClient.scala b/app/models/annotation/WKRemoteTracingStoreClient.scala index 3ee30770dc1..b66ccee76b6 100644 --- a/app/models/annotation/WKRemoteTracingStoreClient.scala +++ b/app/models/annotation/WKRemoteTracingStoreClient.scala @@ -127,17 +127,7 @@ class WKRemoteTracingStoreClient( downsample: Boolean = false, editPosition: Option[Vec3Int] = None, editRotation: Option[Vec3Double] = None, - boundingBox: Option[BoundingBox] = None): Fox[String] = { - logger.debug(s"Called to duplicate volume tracing $volumeTracingId. $baseInfo") - rpc(s"${tracingStore.url}/tracings/volume/$volumeTracingId/duplicate").withLongTimeout - .addQueryString("token" -> RpcTokenHolder.webknossosToken) - .addQueryString("fromTask" -> isFromTask.toString) - .addQueryStringOptional("editPosition", editPosition.map(_.toUriLiteral)) - .addQueryStringOptional("editRotation", editRotation.map(_.toUriLiteral)) - .addQueryStringOptional("boundingBox", boundingBox.map(_.toLiteral)) - .addQueryString("downsample" -> downsample.toString) - .postJsonWithJsonResponse[Option[BoundingBox], String](datasetBoundingBox) - } + boundingBox: Option[BoundingBox] = None): Fox[String] = ??? def mergeSkeletonTracingsByIds(tracingIds: List[String], persistTracing: Boolean): Fox[String] = { logger.debug("Called to merge SkeletonTracings by ids." + baseInfo) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala index f7108ffbaad..a251d545fee 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala @@ -11,12 +11,10 @@ trait AnnotationReversion { def volumeTracingService: VolumeTracingService - def revertDistributedElements(annotationId: String, - currentAnnotationWithTracings: AnnotationWithTracings, + def revertDistributedElements(currentAnnotationWithTracings: AnnotationWithTracings, sourceAnnotationWithTracings: AnnotationWithTracings, revertAction: RevertToVersionAnnotationAction, newVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Unit] = - // TODO segment index, volume buckets, proofreading data for { _ <- Fox.serialCombined(sourceAnnotationWithTracings.getVolumes) { // Only volume data for volume layers present in the *source annotation* needs to be reverted. diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala index 4d44bcc1407..95a35cd4c95 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala @@ -1,5 +1,6 @@ package com.scalableminds.webknossos.tracingstore.annotation +import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayer import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType.AnnotationLayerType import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis import com.scalableminds.webknossos.tracingstore.tracings.volume.MagRestrictions @@ -12,7 +13,9 @@ case class AnnotationLayerParameters(typ: AnnotationLayerType, mappingName: Option[String] = None, magRestrictions: Option[MagRestrictions], name: Option[String], - additionalAxes: Option[Seq[AdditionalAxis]]) + additionalAxes: Option[Seq[AdditionalAxis]]) { + def getNameWithDefault: String = name.getOrElse(AnnotationLayer.defaultNameForType(typ)) +} object AnnotationLayerParameters { implicit val jsonFormat: OFormat[AnnotationLayerParameters] = Json.using[WithDefaultValues].format[AnnotationLayerParameters] diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index fddeda5bf6b..482ebb5b12c 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -92,7 +92,8 @@ case class AnnotationWithTracings( ) def deleteTracing(a: DeleteLayerAnnotationAction): AnnotationWithTracings = - this.copy(annotation = annotation.copy(layers = annotation.layers.filter(_.tracingId != a.tracingId))) + this.copy(annotation = annotation.copy(layers = annotation.layers.filter(_.tracingId != a.tracingId)), + tracingsById = tracingsById.removed(a.tracingId)) def updateLayerMetadata(a: UpdateLayerMetadataAnnotationAction): AnnotationWithTracings = this.copy(annotation = annotation.copy(layers = annotation.layers.map(l => diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 3d96cbe2ede..39ccdb92d0a 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -5,7 +5,7 @@ import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int} import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox -import com.scalableminds.util.tools.Fox.{box2Fox, option2Fox} +import com.scalableminds.util.tools.Fox.{bool2Fox, box2Fox, option2Fox} import com.scalableminds.webknossos.datastore.Annotation.{ AnnotationLayerProto, AnnotationLayerTypeProto, @@ -15,6 +15,7 @@ import com.scalableminds.webknossos.datastore.EditableMappingInfo.EditableMappin import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits +import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayerType import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ EditableMappingLayer, EditableMappingService, @@ -145,6 +146,12 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss targetVersion: Long)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = for { tracingId <- action.tracingId.toFox ?~> "add layer action has no tracingId" + _ <- bool2Fox( + !annotationWithTracings.annotation.layers + .exists(_.name == action.layerParameters.getNameWithDefault)) ?~> "addLayer.nameInUse" + _ <- bool2Fox( + !annotationWithTracings.annotation.layers.exists( + _.`type` == AnnotationLayerTypeProto.skeleton && action.layerParameters.typ == AnnotationLayerType.Skeleton)) ?~> "addLayer.onlyOneSkeletonAllowed" tracing <- remoteWebknossosClient.createTracingFor(annotationId, action.layerParameters, previousVersion = targetVersion - 1) @@ -157,7 +164,6 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss revertAction: RevertToVersionAnnotationAction, newVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = // Note: works only after “ironing out” the update action groups - // TODO: read old annotationProto, tracing, buckets, segment indeces for { sourceAnnotation: AnnotationWithTracings <- getWithTracings( annotationId, @@ -167,14 +173,9 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss requestAll = true) // TODO do we need to request the others? _ = logger.info( s"reverting to suorceVersion ${revertAction.sourceVersion}. got sourceAnnotation with version ${sourceAnnotation.version} with ${sourceAnnotation.skeletonStats}") - _ <- revertDistributedElements(annotationId, annotationWithTracings, sourceAnnotation, revertAction, newVersion) + _ <- revertDistributedElements(annotationWithTracings, sourceAnnotation, revertAction, newVersion) } yield sourceAnnotation - def createTracing(a: AddLayerAnnotationAction)( - implicit ec: ExecutionContext): Fox[Either[SkeletonTracing, VolumeTracing]] = - Fox.failure("not implemented") - // TODO create tracing object (ask wk for needed parameters e.g. fallback layer info?) - def updateActionLog(annotationId: String, newestVersion: Long, oldestVersion: Long)( implicit ec: ExecutionContext): Fox[JsValue] = { def versionedTupleToJson(tuple: (Long, List[UpdateAction])): JsObject = @@ -256,7 +257,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss tracing <- annotation.getEditableMappingInfo(tracingId) ?~> "getEditableMapping.failed" } yield tracing - // move the functions that construct the AnnotationWithTracigns elsewhere? + // TODO move the functions that construct the AnnotationWithTracigns elsewhere to keep this file smaller? private def addEditableMapping( annotationId: String, annotationWithTracings: AnnotationWithTracings, @@ -295,7 +296,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss annotationWithTracings, updatesFlat, annotation.version, - targetVersion) // TODO: targetVersion should be set per update group + targetVersion) // TODO: targetVersion must be set per update group, as reverts may come between these updated <- applyUpdatesGrouped(annotationWithTracingsAndMappings, annotationId, updatesGroupsRegrouped, @@ -716,7 +717,6 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss volumeTracingService.duplicateVolumeData(sourceTracingId, sourceTracing, newTracingId, newTracing)) _ <- Fox.runIf(newTracing.getHasEditableMapping)( duplicateEditableMapping(annotationId, sourceTracingId, newTracingId, version)) - // TODO downsample? } yield newTracingId } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index ea689011306..2058d2d6e23 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -34,9 +34,7 @@ class TSAnnotationController @Inject()( log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { for { - // TODO assert id does not already exist _ <- tracingDataStore.annotations.put(annotationId, 0L, request.body) - _ = logger.info(s"stored annotationProto for $annotationId") } yield Ok } } @@ -109,7 +107,6 @@ class TSAnnotationController @Inject()( isFromTask: Boolean, minMag: Option[Int], maxMag: Option[Int], - downsample: Option[Boolean], // TODO remove or implement editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]): Action[AnyContent] = diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 5caca298037..e5c6ec7a9a1 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -11,7 +11,7 @@ POST /annotation/:annotationId/update GET /annotation/:annotationId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionLog(annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]) GET /annotation/:annotationId/updateActionStatistics @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionStatistics(annotationId: String) GET /annotation/:annotationId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.newestVersion(annotationId: String) -POST /annotation/:annotationId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.duplicate(annotationId: String, newAnnotationId: String, version: Option[Long], isFromTask: Boolean, minMag: Option[Int], maxMag: Option[Int], downsample: Option[Boolean], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) +POST /annotation/:annotationId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.duplicate(annotationId: String, newAnnotationId: String, version: Option[Long], isFromTask: Boolean, minMag: Option[Int], maxMag: Option[Int], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) # Volume tracings POST /volume/save @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.save() From 8a0d637f60e0f75a1e0dfb0a054c47b90465696e Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 28 Oct 2024 11:25:06 +0100 Subject: [PATCH 127/150] use new annotation duplicate in task assignment --- app/models/annotation/Annotation.scala | 12 +++ app/models/annotation/AnnotationService.scala | 81 ++++++++++--------- 2 files changed, 55 insertions(+), 38 deletions(-) diff --git a/app/models/annotation/Annotation.scala b/app/models/annotation/Annotation.scala index d608250a649..f4eccd723db 100755 --- a/app/models/annotation/Annotation.scala +++ b/app/models/annotation/Annotation.scala @@ -515,6 +515,18 @@ class AnnotationDAO @Inject()(sqlClient: SqlClient, annotationLayerDAO: Annotati AND a.typ = ${AnnotationType.Task} """.as[ObjectId]) } yield r.toList + def findBaseIdForTask(taskId: ObjectId)(implicit ctx: DBAccessContext): Fox[ObjectId] = + for { + accessQuery <- readAccessQuery + r <- run(q"""SELECT _id + FROM $existingCollectionName + WHERE _task = $taskId + AND typ = ${AnnotationType.TracingBase} + AND state != ${AnnotationState.Cancelled} + AND $accessQuery""".as[ObjectId]) + firstRow <- r.headOption + } yield firstRow + def findAllByTaskIdAndType(taskId: ObjectId, typ: AnnotationType)( implicit ctx: DBAccessContext): Fox[List[Annotation]] = for { diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index fc2c4bb1c42..bade14cfdf5 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -354,46 +354,37 @@ class AnnotationService @Inject()( def annotationsFor(taskId: ObjectId)(implicit ctx: DBAccessContext): Fox[List[Annotation]] = annotationDAO.findAllByTaskIdAndType(taskId, AnnotationType.Task) - private def tracingsFromBase(annotationBase: Annotation, dataset: Dataset)( - implicit ctx: DBAccessContext, - m: MessagesProvider): Fox[(Option[String], Option[String])] = - for { - _ <- bool2Fox(dataset.isUsable) ?~> Messages("dataset.notImported", dataset.name) - tracingStoreClient <- tracingStoreService.clientFor(dataset) - baseSkeletonIdOpt <- annotationBase.skeletonTracingId - baseVolumeIdOpt <- annotationBase.volumeTracingId - newSkeletonId: Option[String] <- Fox.runOptional(baseSkeletonIdOpt)(skeletonId => - tracingStoreClient.duplicateSkeletonTracing(skeletonId)) - newVolumeId: Option[String] <- Fox.runOptional(baseVolumeIdOpt)(volumeId => - tracingStoreClient.duplicateVolumeTracing(volumeId)) - } yield (newSkeletonId, newVolumeId) - def createAnnotationFor(user: User, taskId: ObjectId, initializingAnnotationId: ObjectId)( implicit m: MessagesProvider, - ctx: DBAccessContext): Fox[Annotation] = { - def useAsTemplateAndInsert(annotation: Annotation) = - for { - datasetName <- datasetDAO.getNameById(annotation._dataset)(GlobalAccessContext) ?~> "dataset.notFoundForAnnotation" - dataset <- datasetDAO.findOne(annotation._dataset) ?~> Messages("dataset.noAccess", datasetName) - (newSkeletonId, newVolumeId) <- tracingsFromBase(annotation, dataset) ?~> s"Failed to use annotation base as template for task $taskId with annotation base ${annotation._id}" - annotationLayers <- AnnotationLayer.layersFromIds(newSkeletonId, newVolumeId) - newAnnotation = annotation.copy( - _id = initializingAnnotationId, - _user = user._id, - annotationLayers = annotationLayers, - state = Active, - typ = AnnotationType.Task, - created = Instant.now, - modified = Instant.now - ) - _ <- annotationDAO.updateInitialized(newAnnotation) - } yield newAnnotation - + ctx: DBAccessContext): Fox[Annotation] = for { - annotationBase <- baseForTask(taskId) ?~> "Failed to retrieve annotation base." - result <- useAsTemplateAndInsert(annotationBase).toFox - } yield result - } + annotationBaseId <- annotationDAO.findBaseIdForTask(taskId) ?~> "Failed to retrieve annotation base id." + annotationBase <- annotationDAO.findOne(annotationBaseId) ?~> "Failed to retrieve annotation base." + datasetName <- datasetDAO.getNameById(annotationBase._dataset)(GlobalAccessContext) ?~> "dataset.notFoundForAnnotation" + dataset <- datasetDAO.findOne(annotationBase._dataset) ?~> Messages("dataset.noAccess", datasetName) + _ <- bool2Fox(dataset.isUsable) ?~> Messages("dataset.notImported", dataset.name) + tracingStoreClient <- tracingStoreService.clientFor(dataset) + duplicatedAnnotationProto <- tracingStoreClient.duplicateAnnotation( + annotationBaseId, + initializingAnnotationId, + version = None, + isFromTask = false, + editPosition = None, + editRotation = None, + boundingBox = None, + magRestrictions = MagRestrictions.empty + ) + newAnnotation = annotationBase.copy( + _id = initializingAnnotationId, + _user = user._id, + annotationLayers = duplicatedAnnotationProto.layers.map(AnnotationLayer.fromProto).toList, + state = Active, + typ = AnnotationType.Task, + created = Instant.now, + modified = Instant.now + ) + _ <- annotationDAO.updateInitialized(newAnnotation) + } yield newAnnotation def createSkeletonTracingBase(datasetName: String, boundingBox: Option[BoundingBox], @@ -705,7 +696,7 @@ class AnnotationService @Inject()( updated <- annotationInformationProvider.provideAnnotation(typ, id, issuingUser) } yield updated - def resetToBase(annotation: Annotation)(implicit ctx: DBAccessContext, m: MessagesProvider): Fox[Unit] = + def resetToBase(annotation: Annotation)(implicit ctx: DBAccessContext, m: MessagesProvider): Fox[Unit] = // TODO: implement as update action? annotation.typ match { case AnnotationType.Explorational => Fox.failure("annotation.revert.tasksOnly") @@ -731,6 +722,20 @@ class AnnotationService @Inject()( } yield () } + private def tracingsFromBase(annotationBase: Annotation, dataset: Dataset)( + implicit ctx: DBAccessContext, + m: MessagesProvider): Fox[(Option[String], Option[String])] = + for { + _ <- bool2Fox(dataset.isUsable) ?~> Messages("dataset.notImported", dataset.name) + tracingStoreClient <- tracingStoreService.clientFor(dataset) + baseSkeletonIdOpt <- annotationBase.skeletonTracingId + baseVolumeIdOpt <- annotationBase.volumeTracingId + newSkeletonId: Option[String] <- Fox.runOptional(baseSkeletonIdOpt)(skeletonId => + tracingStoreClient.duplicateSkeletonTracing(skeletonId)) + newVolumeId: Option[String] <- Fox.runOptional(baseVolumeIdOpt)(volumeId => + tracingStoreClient.duplicateVolumeTracing(volumeId)) + } yield (newSkeletonId, newVolumeId) + private def settingsFor(annotation: Annotation)(implicit ctx: DBAccessContext) = if (annotation.typ == AnnotationType.Task || annotation.typ == AnnotationType.TracingBase) for { From 3d92ae3bc6f322ba6b4490112a6a5eb4c6234a4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Mon, 28 Oct 2024 13:21:09 +0100 Subject: [PATCH 128/150] move old routes to new update actions --- frontend/javascripts/admin/admin_rest_api.ts | 201 ++++++++---------- .../admin/task/task_create_form_view.tsx | 6 +- .../explorative_annotations_view.tsx | 14 +- .../accessors/skeletontracing_accessor.ts | 13 +- .../oxalis/model/actions/save_actions.ts | 2 +- .../oxalis/model/reducers/save_reducer.ts | 11 +- .../oxalis/model/sagas/annotation_saga.tsx | 23 +- .../oxalis/model/sagas/update_actions.ts | 55 ++++- .../oxalis/model_initialization.ts | 4 +- .../view/action-bar/merge_modal_view.tsx | 4 +- .../view/components/editable_text_label.tsx | 2 +- .../oxalis/view/jobs/train_ai_model.tsx | 13 +- .../left-border-tabs/layer_settings_tab.tsx | 78 +++++-- .../modals/add_volume_layer_modal.tsx | 61 ++++-- frontend/javascripts/router.tsx | 6 +- .../backend-snapshot-tests/annotations.e2e.ts | 14 +- .../skeletontracing_server_objects.ts | 13 +- frontend/javascripts/types/api_flow_types.ts | 10 +- 18 files changed, 319 insertions(+), 211 deletions(-) diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts index 44da3577ece..be699cb98b5 100644 --- a/frontend/javascripts/admin/admin_rest_api.ts +++ b/frontend/javascripts/admin/admin_rest_api.ts @@ -1,70 +1,70 @@ import ResumableJS from "resumablejs"; import _ from "lodash"; import dayjs from "dayjs"; -import type { - APIActiveUser, - APIAnnotation, - APIAnnotationInfo, - APIAnnotationType, - APIAnnotationVisibility, - APIAnnotationWithTask, - APIBuildInfo, - APIConnectomeFile, - APIDataSource, - APIDataStore, - APIDataset, - APIDatasetId, - APIFeatureToggles, - APIHistogramData, - APIMapping, - APIMaybeUnimportedDataset, - APIMeshFile, - APIAvailableTasksReport, - APIOrganization, - APIOrganizationCompact, - APIProject, - APIProjectCreator, - APIProjectProgressReport, - APIProjectUpdater, - APIProjectWithStatus, - APIPublication, - APIMagRestrictions, - APIScript, - APIScriptCreator, - APIScriptUpdater, - APITask, - APITaskType, - APITeam, - APITimeInterval, - APITimeTrackingPerAnnotation, - APITimeTrackingSpan, - APITracingStore, - APIUpdateActionBatch, - APIUser, - APIUserLoggedTime, - APIUserTheme, - AnnotationLayerDescriptor, - AnnotationViewConfiguration, - EditableLayerProperties, - ExperienceDomainList, - ServerTracing, - TracingType, - ServerEditableMapping, - APICompoundType, - ZarrPrivateLink, - VoxelyticsWorkflowReport, - VoxelyticsChunkStatistics, - ShortLink, - VoxelyticsWorkflowListing, - APIPricingPlanStatus, - VoxelyticsLogLine, - APIUserCompact, - APIDatasetCompact, - MaintenanceInfo, - AdditionalCoordinate, - LayerLink, - VoxelSize, - APITimeTrackingPerUser, +import { + type APIActiveUser, + type APIAnnotation, + type APIAnnotationInfo, + type APIAnnotationType, + type APIAnnotationVisibility, + type APIAnnotationWithTask, + type APIBuildInfo, + type APIConnectomeFile, + type APIDataSource, + type APIDataStore, + type APIDataset, + type APIDatasetId, + type APIFeatureToggles, + type APIHistogramData, + type APIMapping, + type APIMaybeUnimportedDataset, + type APIMeshFile, + type APIAvailableTasksReport, + type APIOrganization, + type APIOrganizationCompact, + type APIProject, + type APIProjectCreator, + type APIProjectProgressReport, + type APIProjectUpdater, + type APIProjectWithStatus, + type APIPublication, + type APIMagRestrictions, + type APIScript, + type APIScriptCreator, + type APIScriptUpdater, + type APITask, + type APITaskType, + type APITeam, + type APITimeInterval, + type APITimeTrackingPerAnnotation, + type APITimeTrackingSpan, + type APITracingStore, + type APIUpdateActionBatch, + type APIUser, + type APIUserLoggedTime, + type APIUserTheme, + type AnnotationLayerDescriptor, + type AnnotationViewConfiguration, + type ExperienceDomainList, + type ServerTracing, + type TracingType, + type ServerEditableMapping, + type APICompoundType, + type ZarrPrivateLink, + type VoxelyticsWorkflowReport, + type VoxelyticsChunkStatistics, + type ShortLink, + type VoxelyticsWorkflowListing, + type APIPricingPlanStatus, + type VoxelyticsLogLine, + type APIUserCompact, + type APIDatasetCompact, + type MaintenanceInfo, + type AdditionalCoordinate, + type LayerLink, + type VoxelSize, + type APITimeTrackingPerUser, + AnnotationLayerType, } from "types/api_flow_types"; import { APIAnnotationTypeEnum } from "types/api_flow_types"; import type { LOG_LEVELS, Vector2, Vector3 } from "oxalis/constants"; @@ -640,25 +640,8 @@ export function setOthersMayEditForAnnotation( ); } -export function updateAnnotationLayer( - annotationId: string, - annotationType: APIAnnotationType, - tracingId: string, - layerProperties: EditableLayerProperties, -): Promise<{ - name: string | null | undefined; -}> { - return Request.sendJSONReceiveJSON( - `/api/annotations/${annotationType}/${annotationId}/editLayer/${tracingId}`, - { - method: "PATCH", - data: layerProperties, - }, - ); -} - type AnnotationLayerCreateDescriptor = { - typ: "Skeleton" | "Volume"; + typ: AnnotationLayerType; name: string | null | undefined; autoFallbackLayer?: boolean; fallbackLayerName?: string | null | undefined; @@ -666,20 +649,6 @@ type AnnotationLayerCreateDescriptor = { magRestrictions?: APIMagRestrictions | null | undefined; }; -export function addAnnotationLayer( - annotationId: string, - annotationType: APIAnnotationType, - newAnnotationLayer: AnnotationLayerCreateDescriptor, -): Promise { - return Request.sendJSONReceiveJSON( - `/api/annotations/${annotationType}/${annotationId}/addAnnotationLayer`, - { - method: "PATCH", - data: newAnnotationLayer, - }, - ); -} - export function deleteAnnotationLayer( annotationId: string, annotationType: APIAnnotationType, @@ -749,7 +718,7 @@ export function duplicateAnnotation( }); } -export async function getAnnotationInformation( +export async function getMaybeOutdatedAnnotationInformation( annotationId: string, options: RequestOptions = {}, ): Promise { @@ -762,6 +731,19 @@ export async function getAnnotationInformation( return annotation; } +export async function getNewestAnnotationInformation( + annotationId: string, + tracingstoreUrl: string, +): Promise { + const infoUrl = `${tracingstoreUrl}/tracings/annotation/${annotationId}`; + const annotationWithMessages = await Request.receiveJSON(infoUrl); // TODO adjust return type and implement proto type in frontend + + // Extract the potential messages property before returning the task to avoid + // failing e2e tests in annotations.e2e.ts + const { messages: _messages, ...annotation } = annotationWithMessages; + return annotation; +} + export async function getAnnotationCompoundInformation( annotationId: string, annotationType: APICompoundType, @@ -802,14 +784,14 @@ export function createExplorational( if (typ === "skeleton") { layers = [ { - typ: "Skeleton", + typ: AnnotationLayerType.Skeleton, name: "Skeleton", }, ]; } else if (typ === "volume") { layers = [ { - typ: "Volume", + typ: AnnotationLayerType.Volume, name: fallbackLayerName, fallbackLayerName, autoFallbackLayer, @@ -820,11 +802,11 @@ export function createExplorational( } else { layers = [ { - typ: "Skeleton", + typ: AnnotationLayerType.Skeleton, name: "Skeleton", }, { - typ: "Volume", + typ: AnnotationLayerType.Volume, name: fallbackLayerName, fallbackLayerName, autoFallbackLayer, @@ -841,7 +823,9 @@ export async function getTracingsForAnnotation( annotation: APIAnnotation, version: number | null | undefined, ): Promise> { - const skeletonLayers = annotation.annotationLayers.filter((layer) => layer.typ === "Skeleton"); + const skeletonLayers = annotation.annotationLayers.filter( + (layer) => layer.typ === AnnotationLayerType.Skeleton, + ); const fullAnnotationLayers = await Promise.all( annotation.annotationLayers.map((layer) => getTracingForAnnotationType(annotation, layer, version), @@ -872,7 +856,7 @@ export async function acquireAnnotationMutex( export async function getTracingForAnnotationType( annotation: APIAnnotation, annotationLayerDescriptor: AnnotationLayerDescriptor, - version?: number | null | undefined, // TODO: Use this parameter + version?: number | null | undefined, // TODOM: Use this parameter ): Promise { const { tracingId, typ } = annotationLayerDescriptor; const tracingType = typ.toLowerCase() as "skeleton" | "volume"; @@ -1001,17 +985,6 @@ export async function importVolumeTracing( ); } -export function convertToHybridTracing( - annotationId: string, - fallbackLayerName: string | null | undefined, -): Promise { - return Request.receiveJSON(`/api/annotations/Explorational/${annotationId}/makeHybrid`, { - method: "PATCH", - // @ts-expect-error ts-migrate(2345) FIXME: Argument of type '{ method: "PATCH"; fallbackLayer... Remove this comment to see the full error message - fallbackLayerName, - }); -} - export async function downloadWithFilename(downloadUrl: string) { const link = document.createElement("a"); link.href = downloadUrl; diff --git a/frontend/javascripts/admin/task/task_create_form_view.tsx b/frontend/javascripts/admin/task/task_create_form_view.tsx index 075086defad..8b90128c6dc 100644 --- a/frontend/javascripts/admin/task/task_create_form_view.tsx +++ b/frontend/javascripts/admin/task/task_create_form_view.tsx @@ -35,7 +35,7 @@ import { createTaskFromNML, createTasks, getActiveDatasetsOfMyOrganization, - getAnnotationInformation, + getMaybeOutdatedAnnotationInformation, getProjects, getScripts, getTask, @@ -481,12 +481,12 @@ function TaskCreateFormView({ taskId, history }: Props) { const annotationResponse = (await tryToAwaitPromise( - getAnnotationInformation(value, { + getMaybeOutdatedAnnotationInformation(value, { showErrorToast: false, }), )) || (await tryToAwaitPromise( - getAnnotationInformation(value, { + getMaybeOutdatedAnnotationInformation(value, { showErrorToast: false, }), )); diff --git a/frontend/javascripts/dashboard/explorative_annotations_view.tsx b/frontend/javascripts/dashboard/explorative_annotations_view.tsx index 6710eba5b37..6378e6d6511 100644 --- a/frontend/javascripts/dashboard/explorative_annotations_view.tsx +++ b/frontend/javascripts/dashboard/explorative_annotations_view.tsx @@ -68,6 +68,8 @@ import { ActiveTabContext, RenderingTabContext } from "./dashboard_contexts"; import type { SearchProps } from "antd/lib/input"; import { getCombinedStatsFromServerAnnotation } from "oxalis/model/accessors/annotation_accessor"; import { AnnotationStats } from "oxalis/view/right-border-tabs/dataset_info_tab_view"; +import { pushSaveQueueTransaction } from "oxalis/model/actions/save_actions"; +import { updateMetadataOfAnnotation } from "oxalis/model/sagas/update_actions"; const { Search } = Input; const pageLength: number = 1000; @@ -384,14 +386,10 @@ class ExplorativeAnnotationsView extends React.PureComponent { }; renameTracing(tracing: APIAnnotationInfo, name: string) { - editAnnotation(tracing.id, tracing.typ, { name }) - .then(() => { - Toast.success(messages["annotation.was_edited"]); - this.updateTracingInLocalState(tracing, (t) => update(t, { name: { $set: name } })); - }) - .catch((error) => { - handleGenericError(error as Error, "Could not update the annotation name."); - }); + Store.dispatch( + pushSaveQueueTransaction([updateMetadataOfAnnotation(name)], "unused-tracing-id"), + ); + this.updateTracingInLocalState(tracing, (t) => update(t, { name: { $set: name } })); } archiveAll = () => { diff --git a/frontend/javascripts/oxalis/model/accessors/skeletontracing_accessor.ts b/frontend/javascripts/oxalis/model/accessors/skeletontracing_accessor.ts index b30708bb6c9..f668fc3f7c5 100644 --- a/frontend/javascripts/oxalis/model/accessors/skeletontracing_accessor.ts +++ b/frontend/javascripts/oxalis/model/accessors/skeletontracing_accessor.ts @@ -1,10 +1,11 @@ import Maybe from "data.maybe"; import _ from "lodash"; -import type { - ServerTracing, - ServerSkeletonTracing, - APIAnnotation, - AnnotationLayerDescriptor, +import { + type ServerTracing, + type ServerSkeletonTracing, + type APIAnnotation, + type AnnotationLayerDescriptor, + AnnotationLayerType, } from "types/api_flow_types"; import type { Tracing, @@ -41,7 +42,7 @@ export function getSkeletonDescriptor( annotation: APIAnnotation, ): AnnotationLayerDescriptor | null | undefined { const skeletonLayers = annotation.annotationLayers.filter( - (descriptor) => descriptor.typ === "Skeleton", + (descriptor) => descriptor.typ === AnnotationLayerType.Skeleton, ); if (skeletonLayers.length > 0) { diff --git a/frontend/javascripts/oxalis/model/actions/save_actions.ts b/frontend/javascripts/oxalis/model/actions/save_actions.ts index 870ba1f730f..64f2c04eadc 100644 --- a/frontend/javascripts/oxalis/model/actions/save_actions.ts +++ b/frontend/javascripts/oxalis/model/actions/save_actions.ts @@ -36,8 +36,8 @@ export const pushSaveQueueTransaction = ( ({ type: "PUSH_SAVE_QUEUE_TRANSACTION", items, - transactionId, tracingId, + transactionId, }) as const; export const saveNowAction = () => diff --git a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts index 73de1df5aca..2bfd14c7f49 100644 --- a/frontend/javascripts/oxalis/model/reducers/save_reducer.ts +++ b/frontend/javascripts/oxalis/model/reducers/save_reducer.ts @@ -193,12 +193,21 @@ function SaveReducer(state: OxalisState, action: Action): OxalisState { } } +const layerIndependentActions = new Set([ + "updateTdCamera", + "revertToVersion", + "addLayerToAnnotation", + "deleteLayerFromAnnotation", + "updateLayerMetadata", + "updateMetadataOfAnnotation", +]); + export function addTracingIdToActions( actions: UpdateAction[], tracingId: string, ): Array { return actions.map((action) => { - if (action.name === "updateTdCamera" || action.name === "revertToVersion") { + if (layerIndependentActions.has(action.name)) { return action as UpdateAction; } return { diff --git a/frontend/javascripts/oxalis/model/sagas/annotation_saga.tsx b/frontend/javascripts/oxalis/model/sagas/annotation_saga.tsx index c93961d49cd..2b036703d5c 100644 --- a/frontend/javascripts/oxalis/model/sagas/annotation_saga.tsx +++ b/frontend/javascripts/oxalis/model/sagas/annotation_saga.tsx @@ -9,11 +9,7 @@ import { } from "oxalis/model/actions/annotation_actions"; import type { EditableAnnotation } from "admin/admin_rest_api"; import type { ActionPattern } from "redux-saga/effects"; -import { - editAnnotation, - updateAnnotationLayer, - acquireAnnotationMutex, -} from "admin/admin_rest_api"; +import { editAnnotation, acquireAnnotationMutex } from "admin/admin_rest_api"; import { SETTINGS_MAX_RETRY_COUNT, SETTINGS_RETRY_DELAY, @@ -47,6 +43,8 @@ import { determineLayout } from "oxalis/view/layouting/default_layout_configs"; import { getLastActiveLayout, getLayoutConfig } from "oxalis/view/layouting/layout_persistence"; import { is3dViewportMaximized } from "oxalis/view/layouting/flex_layout_helper"; import { needsLocalHdf5Mapping } from "../accessors/volumetracing_accessor"; +import { pushSaveQueueTransaction } from "../actions/save_actions"; +import { updateAnnotationLayerName } from "./update_actions"; /* Note that this must stay in sync with the back-end constant MaxMagForAgglomerateMapping compare https://github.com/scalableminds/webknossos/issues/5223. @@ -103,16 +101,11 @@ export function* pushAnnotationUpdateAsync(action: Action) { function* pushAnnotationLayerUpdateAsync(action: EditAnnotationLayerAction): Saga { const { tracingId, layerProperties } = action; - const annotationId = yield* select((storeState) => storeState.tracing.annotationId); - const annotationType = yield* select((storeState) => storeState.tracing.annotationType); - yield* retry( - SETTINGS_MAX_RETRY_COUNT, - SETTINGS_RETRY_DELAY, - updateAnnotationLayer, - annotationId, - annotationType, - tracingId, - layerProperties, + yield* put( + pushSaveQueueTransaction( + [updateAnnotationLayerName(tracingId, layerProperties.name)], + tracingId, + ), ); } diff --git a/frontend/javascripts/oxalis/model/sagas/update_actions.ts b/frontend/javascripts/oxalis/model/sagas/update_actions.ts index b7ab2a80b87..cbfdc61f9e5 100644 --- a/frontend/javascripts/oxalis/model/sagas/update_actions.ts +++ b/frontend/javascripts/oxalis/model/sagas/update_actions.ts @@ -10,7 +10,11 @@ import type { NumberLike, } from "oxalis/store"; import { convertUserBoundingBoxesFromFrontendToServer } from "oxalis/model/reducers/reducer_helpers"; -import type { AdditionalCoordinate, MetadataEntryProto } from "types/api_flow_types"; +import type { + AdditionalCoordinate, + APIMagRestrictions, + MetadataEntryProto, +} from "types/api_flow_types"; export type NodeWithTreeId = { treeId: number; @@ -51,6 +55,10 @@ export type RevertToVersionUpdateAction = ReturnType; export type RemoveFallbackLayerUpdateAction = ReturnType; export type UpdateTdCameraUpdateAction = ReturnType; export type UpdateMappingNameUpdateAction = ReturnType; +type AddLayerToAnnotationUpdateAction = ReturnType; +type DeleteAnnotationLayerUpdateAction = ReturnType; +type UpdateAnnotationLayerNameUpdateAction = ReturnType; +type UpdateMetadataOfAnnotationUpdateAction = ReturnType; export type SplitAgglomerateUpdateAction = ReturnType; export type MergeAgglomerateUpdateAction = ReturnType; @@ -82,6 +90,10 @@ export type UpdateAction = | RemoveFallbackLayerUpdateAction | UpdateTdCameraUpdateAction | UpdateMappingNameUpdateAction + | AddLayerToAnnotationUpdateAction + | DeleteAnnotationLayerUpdateAction + | UpdateAnnotationLayerNameUpdateAction + | UpdateMetadataOfAnnotationUpdateAction | SplitAgglomerateUpdateAction | MergeAgglomerateUpdateAction; @@ -529,6 +541,47 @@ export function mergeAgglomerate( } as const; } +type AnnotationLayerCreationParameters = { + typ: "Skeleton" | "Volume"; + name: string | null | undefined; + autoFallbackLayer?: boolean; + fallbackLayerName?: string | null | undefined; + mappingName?: string | null | undefined; + magRestrictions?: APIMagRestrictions | null | undefined; +}; + +export function addLayerToAnnotation(parameters: AnnotationLayerCreationParameters) { + return { + name: "addLayerToAnnotation", + value: { layerParameters: parameters }, + } as const; +} + +export function deleteAnnotationLayer( + tracingId: string, + layerName: string, + typ: "Skeleton" | "Volume", +) { + return { + name: "deleteLayerFromAnnotation", + value: { tracingId, layerName, typ }, + } as const; +} + +export function updateAnnotationLayerName(tracingId: string, newLayerName: string) { + return { + name: "updateLayerMetadata", + value: { tracingId, layerName: newLayerName }, + } as const; +} + +export function updateMetadataOfAnnotation(name?: string, description?: string) { + return { + name: "updateMetadataOfAnnotation", + value: { name, description }, + } as const; +} + function enforceValidMetadata(metadata: MetadataEntryProto[]): MetadataEntryProto[] { // We do not want to save metadata with duplicate keys. Validation errors // will warn the user in case this exists. However, we allow duplicate keys in the diff --git a/frontend/javascripts/oxalis/model_initialization.ts b/frontend/javascripts/oxalis/model_initialization.ts index 902216e0d26..c456701fbff 100644 --- a/frontend/javascripts/oxalis/model_initialization.ts +++ b/frontend/javascripts/oxalis/model_initialization.ts @@ -35,7 +35,7 @@ import { getServerVolumeTracings } from "oxalis/model/accessors/volumetracing_ac import { getSomeServerTracing } from "oxalis/model/accessors/tracing_accessor"; import { getTracingsForAnnotation, - getAnnotationInformation, + getMaybeOutdatedAnnotationInformation, getEmptySandboxAnnotationInformation, getDataset, getSharingTokenFromUrlParameters, @@ -132,7 +132,7 @@ export async function initialize( annotation = initialMaybeCompoundType != null ? await getAnnotationCompoundInformation(annotationId, initialMaybeCompoundType) - : await getAnnotationInformation(annotationId); + : await getMaybeOutdatedAnnotationInformation(annotationId); datasetId = { name: annotation.dataSetName, owningOrganization: annotation.organization, diff --git a/frontend/javascripts/oxalis/view/action-bar/merge_modal_view.tsx b/frontend/javascripts/oxalis/view/action-bar/merge_modal_view.tsx index 9f3f5f52d9d..cd09d7c3f84 100644 --- a/frontend/javascripts/oxalis/view/action-bar/merge_modal_view.tsx +++ b/frontend/javascripts/oxalis/view/action-bar/merge_modal_view.tsx @@ -8,7 +8,7 @@ import { addTreesAndGroupsAction } from "oxalis/model/actions/skeletontracing_ac import { getSkeletonDescriptor } from "oxalis/model/accessors/skeletontracing_accessor"; import { createMutableTreeMapFromTreeArray } from "oxalis/model/reducers/skeletontracing_reducer_helpers"; import { - getAnnotationInformation, + getMaybeOutdatedAnnotationInformation, getAnnotationCompoundInformation, getTracingForAnnotationType, } from "admin/admin_rest_api"; @@ -145,7 +145,7 @@ class _MergeModalView extends PureComponent { const { selectedExplorativeAnnotation } = this.state; if (selectedExplorativeAnnotation != null) { - const annotation = await getAnnotationInformation(selectedExplorativeAnnotation); + const annotation = await getMaybeOutdatedAnnotationInformation(selectedExplorativeAnnotation); this.mergeAnnotationIntoActiveTracing(annotation); } }; diff --git a/frontend/javascripts/oxalis/view/components/editable_text_label.tsx b/frontend/javascripts/oxalis/view/components/editable_text_label.tsx index 6b514d5bc2d..443698164f7 100644 --- a/frontend/javascripts/oxalis/view/components/editable_text_label.tsx +++ b/frontend/javascripts/oxalis/view/components/editable_text_label.tsx @@ -14,7 +14,7 @@ type Rule = { }; export type EditableTextLabelProp = { value: string; - onChange: (...args: Array) => any; + onChange: (newValue: string) => any; rules?: Rule[]; rows?: number; markdown?: boolean; diff --git a/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx b/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx index 1153c9eaf0e..51968b6bae6 100644 --- a/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx +++ b/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx @@ -21,7 +21,7 @@ import { getSegmentationLayers, } from "oxalis/model/accessors/dataset_accessor"; import { - getAnnotationInformation, + getMaybeOutdatedAnnotationInformation, getDataset, getTracingForAnnotationType, runTraining, @@ -35,7 +35,12 @@ import BoundingBox from "oxalis/model/bucket_data_handling/bounding_box"; import { formatVoxels } from "libs/format_utils"; import * as Utils from "libs/utils"; import { V3 } from "libs/mjs"; -import type { APIAnnotation, APIDataset, ServerVolumeTracing } from "types/api_flow_types"; +import { + AnnotationLayerType, + type APIAnnotation, + type APIDataset, + type ServerVolumeTracing, +} from "types/api_flow_types"; import type { Vector3 } from "oxalis/constants"; import { serverVolumeToClientVolumeTracing } from "oxalis/model/reducers/volumetracing_reducer"; import { convertUserBoundingBoxesFromServerToFrontend } from "oxalis/model/reducers/reducer_helpers"; @@ -472,7 +477,7 @@ function AnnotationsCsvInput({ const newAnnotationsWithDatasets = await Promise.all( newItems.map(async (item) => { - const annotation = await getAnnotationInformation(item.annotationId); + const annotation = await getMaybeOutdatedAnnotationInformation(item.annotationId); const dataset = await getDataset({ owningOrganization: annotation.organization, name: annotation.dataSetName, @@ -493,7 +498,7 @@ function AnnotationsCsvInput({ let userBoundingBoxes = volumeTracings[0]?.userBoundingBoxes; if (!userBoundingBoxes) { const skeletonLayer = annotation.annotationLayers.find( - (layer) => layer.typ === "Skeleton", + (layer) => layer.typ === AnnotationLayerType.Skeleton, ); if (skeletonLayer) { const skeletonTracing = await getTracingForAnnotationType(annotation, skeletonLayer); diff --git a/frontend/javascripts/oxalis/view/left-border-tabs/layer_settings_tab.tsx b/frontend/javascripts/oxalis/view/left-border-tabs/layer_settings_tab.tsx index 2a3a4b3fe93..bd9ef5bd1a8 100644 --- a/frontend/javascripts/oxalis/view/left-border-tabs/layer_settings_tab.tsx +++ b/frontend/javascripts/oxalis/view/left-border-tabs/layer_settings_tab.tsx @@ -21,6 +21,7 @@ import _ from "lodash"; import classnames from "classnames"; import update from "immutability-helper"; import { + AnnotationLayerType, APIAnnotationTypeEnum, type APIDataLayer, type APIDataset, @@ -49,8 +50,6 @@ import { findDataPositionForLayer, clearCache, findDataPositionForVolumeTracing, - convertToHybridTracing, - deleteAnnotationLayer, updateDatasetDefaultConfiguration, startComputeSegmentIndexFileJob, } from "admin/admin_rest_api"; @@ -131,6 +130,8 @@ import { getDefaultLayerViewConfiguration, } from "types/schemas/dataset_view_configuration.schema"; import defaultState from "oxalis/default_state"; +import { pushSaveQueueTransaction } from "oxalis/model/actions/save_actions"; +import { addLayerToAnnotation, deleteAnnotationLayer } from "oxalis/model/sagas/update_actions"; type DatasetSettingsProps = { userConfiguration: UserConfiguration; @@ -150,6 +151,8 @@ type DatasetSettingsProps = { onZoomToMag: (layerName: string, arg0: Vector3) => number; onChangeUser: (key: keyof UserConfiguration, value: any) => void; reloadHistogram: (layerName: string) => void; + addSkeletonLayerToAnnotation: () => void; + deleteAnnotationLayer: (tracingId: string, type: AnnotationLayerType, layerName: string) => void; tracing: Tracing; task: Task | null | undefined; onEditAnnotationLayer: (tracingId: string, layerProperties: EditableLayerProperties) => void; @@ -453,26 +456,39 @@ class DatasetSettings extends React.PureComponent { ); - getDeleteAnnotationLayerButton = (readableName: string, layer?: APIDataLayer) => ( + getDeleteAnnotationLayerButton = ( + readableName: string, + type: AnnotationLayerType, + tracingId: string, + ) => (
this.deleteAnnotationLayerIfConfirmed(readableName, layer)} + onClick={() => this.deleteAnnotationLayerIfConfirmed(readableName, type, tracingId)} className="fas fa-trash icon-margin-right" />
); - getDeleteAnnotationLayerDropdownOption = (readableName: string, layer?: APIDataLayer) => ( -
this.deleteAnnotationLayerIfConfirmed(readableName, layer)}> + getDeleteAnnotationLayerDropdownOption = ( + readableName: string, + type: AnnotationLayerType, + tracingId: string, + layer?: APIDataLayer, + ) => ( +
this.deleteAnnotationLayerIfConfirmed(readableName, type, tracingId, layer)} + > Delete this annotation layer
); deleteAnnotationLayerIfConfirmed = async ( - readableAnnoationLayerName: string, + readableAnnotationLayerName: string, + type: AnnotationLayerType, + tracingId: string, layer?: APIDataLayer, ) => { const fallbackLayerNote = @@ -481,7 +497,7 @@ class DatasetSettings extends React.PureComponent { : ""; const shouldDelete = await confirmAsync({ title: `Deleting an annotation layer makes its content and history inaccessible. ${fallbackLayerNote}This cannot be undone. Are you sure you want to delete this layer?`, - okText: `Yes, delete annotation layer “${readableAnnoationLayerName}”`, + okText: `Yes, delete annotation layer “${readableAnnotationLayerName}”`, cancelText: "Cancel", maskClosable: true, closable: true, @@ -495,12 +511,8 @@ class DatasetSettings extends React.PureComponent { }, }); if (!shouldDelete) return; + this.props.deleteAnnotationLayer(tracingId, type, readableAnnotationLayerName); await Model.ensureSavedState(); - await deleteAnnotationLayer( - this.props.tracing.annotationId, - this.props.tracing.annotationType, - readableAnnoationLayerName, - ); location.reload(); }; @@ -623,6 +635,8 @@ class DatasetSettings extends React.PureComponent { const { intensityRange } = layerSettings; const layer = getLayerByName(dataset, layerName); const isSegmentation = layer.category === "segmentation"; + const layerType = + layer.category === "segmentation" ? AnnotationLayerType.Volume : AnnotationLayerType.Skeleton; const canBeMadeEditable = isSegmentation && layer.tracingId == null && this.props.controlMode === "TRACE"; const isVolumeTracing = isSegmentation ? layer.tracingId != null : false; @@ -687,7 +701,12 @@ class DatasetSettings extends React.PureComponent { ? { label: (
- {this.getDeleteAnnotationLayerDropdownOption(readableName, layer)} + {this.getDeleteAnnotationLayerDropdownOption( + readableName, + layerType, + layer.tracingId, + layer, + )}
), key: "deleteAnnotationLayer", @@ -1173,7 +1192,7 @@ class DatasetSettings extends React.PureComponent { const readableName = "Skeleton"; const skeletonTracing = enforceSkeletonTracing(tracing); const isOnlyAnnotationLayer = tracing.annotationLayers.length === 1; - const { showSkeletons } = skeletonTracing; + const { showSkeletons, tracingId } = skeletonTracing; const activeNodeRadius = getActiveNode(skeletonTracing)?.radius ?? 0; return ( @@ -1224,7 +1243,13 @@ class DatasetSettings extends React.PureComponent { }} > - {!isOnlyAnnotationLayer ? this.getDeleteAnnotationLayerButton(readableName) : null} + {!isOnlyAnnotationLayer + ? this.getDeleteAnnotationLayerButton( + readableName, + AnnotationLayerType.Skeleton, + tracingId, + ) + : null}
{showSkeletons ? ( @@ -1325,8 +1350,8 @@ class DatasetSettings extends React.PureComponent { }; addSkeletonAnnotationLayer = async () => { + this.props.addSkeletonLayerToAnnotation(); await Model.ensureSavedState(); - await convertToHybridTracing(this.props.tracing.annotationId, null); location.reload(); }; @@ -1639,6 +1664,25 @@ const mapDispatchToProps = (dispatch: Dispatch) => ({ reloadHistogram(layerName: string) { dispatch(reloadHistogramAction(layerName)); }, + + addSkeletonLayerToAnnotation() { + dispatch( + pushSaveQueueTransaction( + [ + addLayerToAnnotation({ + typ: "Skeleton", + name: "skeleton", + fallbackLayerName: undefined, + }), + ], + "unused-tracing-id", + ), + ); + }, + + deleteAnnotationLayer(tracingId: string, type: AnnotationLayerType, layerName: string) { + dispatch(deleteAnnotationLayer(tracingId, layerName, type)); + }, }); const connector = connect(mapStateToProps, mapDispatchToProps); diff --git a/frontend/javascripts/oxalis/view/left-border-tabs/modals/add_volume_layer_modal.tsx b/frontend/javascripts/oxalis/view/left-border-tabs/modals/add_volume_layer_modal.tsx index 565b8ace677..33885d71953 100644 --- a/frontend/javascripts/oxalis/view/left-border-tabs/modals/add_volume_layer_modal.tsx +++ b/frontend/javascripts/oxalis/view/left-border-tabs/modals/add_volume_layer_modal.tsx @@ -10,7 +10,6 @@ import { RestrictMagnificationSlider, } from "dashboard/advanced_dataset/create_explorative_modal"; import Store, { type Tracing } from "oxalis/store"; -import { addAnnotationLayer } from "admin/admin_rest_api"; import { getSomeMagInfoForDataset, getLayerByName, @@ -24,9 +23,12 @@ import { } from "oxalis/model/accessors/volumetracing_accessor"; import messages from "messages"; import InputComponent from "oxalis/view/components/input_component"; -import { api } from "oxalis/singletons"; +import { api, Model } from "oxalis/singletons"; import Toast from "libs/toast"; import { MappingStatusEnum } from "oxalis/constants"; +import { pushSaveQueueTransaction } from "oxalis/model/actions/save_actions"; +import { useDispatch } from "react-redux"; +import { addLayerToAnnotation } from "oxalis/model/sagas/update_actions"; export type ValidationResult = { isValid: boolean; message: string }; export function checkForLayerNameDuplication( @@ -101,6 +103,7 @@ export default function AddVolumeLayerModal({ const [selectedSegmentationLayerName, setSelectedSegmentationLayerName] = useState< string | undefined >(preselectedLayerName); + const dispatch = useDispatch(); const allReadableLayerNames = useMemo( () => getAllReadableLayerNames(dataset, tracing), [dataset, tracing], @@ -162,15 +165,23 @@ export default function AddVolumeLayerModal({ ); if (selectedSegmentationLayerName == null) { - await addAnnotationLayer(tracing.annotationId, tracing.annotationType, { - typ: "Volume", - name: newLayerName, - fallbackLayerName: undefined, - magRestrictions: { - min: minResolutionAllowed, - max: maxResolutionAllowed, - }, - }); + dispatch( + pushSaveQueueTransaction( + [ + addLayerToAnnotation({ + typ: "Volume", + name: newLayerName, + fallbackLayerName: undefined, + magRestrictions: { + min: minResolutionAllowed, + max: maxResolutionAllowed, + }, + }), + ], + "unused-tracing-id", + ), + ); + await Model.ensureSavedState(); } else { if (selectedSegmentationLayer == null) { throw new Error("Segmentation layer is null"); @@ -189,16 +200,24 @@ export default function AddVolumeLayerModal({ maybeMappingName = mappingInfo.mappingName; } - await addAnnotationLayer(tracing.annotationId, tracing.annotationType, { - typ: "Volume", - name: newLayerName, - fallbackLayerName, - magRestrictions: { - min: minResolutionAllowed, - max: maxResolutionAllowed, - }, - mappingName: maybeMappingName, - }); + dispatch( + pushSaveQueueTransaction( + [ + addLayerToAnnotation({ + typ: "Volume", + name: newLayerName, + fallbackLayerName, + magRestrictions: { + min: minResolutionAllowed, + max: maxResolutionAllowed, + }, + mappingName: maybeMappingName, + }), + ], + "unused-tracing-id", + ), + ); + await Model.ensureSavedState(); } await api.tracing.hardReload(); diff --git a/frontend/javascripts/router.tsx b/frontend/javascripts/router.tsx index dcbc7815c49..178958aba06 100644 --- a/frontend/javascripts/router.tsx +++ b/frontend/javascripts/router.tsx @@ -1,6 +1,6 @@ import { createExplorational, - getAnnotationInformation, + getMaybeOutdatedAnnotationInformation, getOrganizationForDataset, getShortLink, } from "admin/admin_rest_api"; @@ -198,7 +198,9 @@ class ReactRouter extends React.Component { serverAuthenticationCallback = async ({ match }: ContextRouter) => { try { - const annotationInformation = await getAnnotationInformation(match.params.id || ""); + const annotationInformation = await getMaybeOutdatedAnnotationInformation( + match.params.id || "", + ); return annotationInformation.visibility === "Public"; } catch (_ex) { // Annotation could not be found diff --git a/frontend/javascripts/test/backend-snapshot-tests/annotations.e2e.ts b/frontend/javascripts/test/backend-snapshot-tests/annotations.e2e.ts index 3e0833db14f..ffb9b0243c7 100644 --- a/frontend/javascripts/test/backend-snapshot-tests/annotations.e2e.ts +++ b/frontend/javascripts/test/backend-snapshot-tests/annotations.e2e.ts @@ -35,7 +35,7 @@ test.before("Reset database", async () => { }); test("getAnnotationInformation()", async (t) => { const annotationId = "570ba0092a7c0e980056fe9b"; - const annotation = await api.getAnnotationInformation(annotationId); + const annotation = await api.getMaybeOutdatedAnnotationInformation(annotationId); t.is(annotation.id, annotationId); writeTypeCheckingFile(annotation, "annotation", "APIAnnotation"); t.snapshot(annotation); @@ -43,7 +43,7 @@ test("getAnnotationInformation()", async (t) => { test("getAnnotationInformation() for public annotation while logged out", async (t) => { setCurrToken("invalidToken"); const annotationId = "88135c192faeb34c0081c05d"; - const annotation = await api.getAnnotationInformation(annotationId); + const annotation = await api.getMaybeOutdatedAnnotationInformation(annotationId); t.is(annotation.id, annotationId); t.snapshot(annotation); setCurrToken(tokenUserA); @@ -78,7 +78,7 @@ test.serial("finishAnnotation() and reOpenAnnotation() for explorational", async }); test.serial("editAnnotation()", async (t) => { const annotationId = "68135c192faeb34c0081c05d"; - const originalAnnotation = await api.getAnnotationInformation(annotationId); + const originalAnnotation = await api.getMaybeOutdatedAnnotationInformation(annotationId); const { name, visibility, description } = originalAnnotation; const newName = "new name"; const newVisibility = "Public"; @@ -88,7 +88,7 @@ test.serial("editAnnotation()", async (t) => { visibility: newVisibility, description: newDescription, }); - const editedAnnotation = await api.getAnnotationInformation(annotationId); + const editedAnnotation = await api.getMaybeOutdatedAnnotationInformation(annotationId); t.is(editedAnnotation.name, newName); t.is(editedAnnotation.visibility, newVisibility); t.is(editedAnnotation.description, newDescription); @@ -106,7 +106,7 @@ test.serial("finishAllAnnotations()", async (t) => { const annotationIds = ["78135c192faeb34c0081c05d", "78135c192faeb34c0081c05e"]; await api.finishAllAnnotations(annotationIds); const finishedAnnotations = await Promise.all( - annotationIds.map((id) => api.getAnnotationInformation(id)), + annotationIds.map((id) => api.getMaybeOutdatedAnnotationInformation(id)), ); t.is(finishedAnnotations.length, 2); finishedAnnotations.forEach((annotation) => { @@ -120,7 +120,9 @@ test.serial("createExplorational() and finishAnnotation()", async (t) => { const createdExplorational = await api.createExplorational(datasetId, "skeleton", false, null); t.snapshot(replaceVolatileValues(createdExplorational)); await api.finishAnnotation(createdExplorational.id, APIAnnotationTypeEnum.Explorational); - const finishedAnnotation = await api.getAnnotationInformation(createdExplorational.id); + const finishedAnnotation = await api.getMaybeOutdatedAnnotationInformation( + createdExplorational.id, + ); t.is(finishedAnnotation.state, "Finished"); }); test.serial("getTracingsForAnnotation()", async (t) => { diff --git a/frontend/javascripts/test/fixtures/skeletontracing_server_objects.ts b/frontend/javascripts/test/fixtures/skeletontracing_server_objects.ts index 55a2c1fa71d..b2c4d8db95b 100644 --- a/frontend/javascripts/test/fixtures/skeletontracing_server_objects.ts +++ b/frontend/javascripts/test/fixtures/skeletontracing_server_objects.ts @@ -1,6 +1,10 @@ -import type { ServerSkeletonTracing, APIAnnotation } from "types/api_flow_types"; +import { + type ServerSkeletonTracing, + type APIAnnotation, + AnnotationLayerType, +} from "types/api_flow_types"; export const tracing: ServerSkeletonTracing = { - typ: "Skeleton", + typ: AnnotationLayerType.Skeleton, id: "47e37793-d0be-4240-a371-87ce68561a13", trees: [ { @@ -173,11 +177,12 @@ export const annotation: APIAnnotation = { allowDownload: true, allowSave: true, }, + version: 0, annotationLayers: [ { - name: "Skeleton", + name: AnnotationLayerType.Skeleton, tracingId: "47e37793-d0be-4240-a371-87ce68561a13", - typ: "Skeleton", + typ: AnnotationLayerType.Skeleton, stats: {}, }, ], diff --git a/frontend/javascripts/types/api_flow_types.ts b/frontend/javascripts/types/api_flow_types.ts index 4de739d10b4..42fcf83ce52 100644 --- a/frontend/javascripts/types/api_flow_types.ts +++ b/frontend/javascripts/types/api_flow_types.ts @@ -385,6 +385,10 @@ export enum TracingTypeEnum { volume = "volume", hybrid = "hybrid", } +export enum AnnotationLayerType { + Skeleton = "Skeleton", + Volume = "Volume", +} export type TracingType = keyof typeof TracingTypeEnum; export type APITaskType = { readonly id: string; @@ -467,12 +471,12 @@ export type APITask = { export type AnnotationLayerDescriptor = { name: string; tracingId: string; - typ: "Skeleton" | "Volume"; + typ: AnnotationLayerType; stats: TracingStats | EmptyObject; }; -export type EditableLayerProperties = Partial<{ +export type EditableLayerProperties = { name: string; -}>; +}; export type APIAnnotationInfo = { readonly annotationLayers: Array; readonly dataSetName: string; From 41cada12b4199b56db8cb11447808c07c50c66a7 Mon Sep 17 00:00:00 2001 From: Florian M Date: Mon, 28 Oct 2024 16:57:34 +0100 Subject: [PATCH 129/150] spelling of layer type proto --- .../models/annotation/AnnotationLayerType.scala | 8 ++++---- webknossos-datastore/proto/Annotation.proto | 4 ++-- .../tracingstore/annotation/TSAnnotationService.scala | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayerType.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayerType.scala index 9b1a7dd2d9d..2593bedce4f 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayerType.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayerType.scala @@ -9,14 +9,14 @@ object AnnotationLayerType extends ExtendedEnumeration { def toProto(annotationLayerType: AnnotationLayerType): AnnotationLayerTypeProto = annotationLayerType match { - case Skeleton => AnnotationLayerTypeProto.skeleton - case Volume => AnnotationLayerTypeProto.volume + case Skeleton => AnnotationLayerTypeProto.Skeleton + case Volume => AnnotationLayerTypeProto.Volume } def fromProto(p: AnnotationLayerTypeProto): AnnotationLayerType = p match { - case AnnotationLayerTypeProto.skeleton => Skeleton - case AnnotationLayerTypeProto.volume => Volume + case AnnotationLayerTypeProto.Skeleton => Skeleton + case AnnotationLayerTypeProto.Volume => Volume case AnnotationLayerTypeProto.Unrecognized(_) => Volume // unrecognized should never happen, artifact of proto code generation } diff --git a/webknossos-datastore/proto/Annotation.proto b/webknossos-datastore/proto/Annotation.proto index 4fe56262b5c..e8938373307 100644 --- a/webknossos-datastore/proto/Annotation.proto +++ b/webknossos-datastore/proto/Annotation.proto @@ -3,8 +3,8 @@ syntax = "proto2"; package com.scalableminds.webknossos.datastore; enum AnnotationLayerTypeProto { - skeleton = 1; - volume = 2; + Skeleton = 1; + Volume = 2; } message AnnotationProto { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 39ccdb92d0a..627e4b640c7 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -151,7 +151,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss .exists(_.name == action.layerParameters.getNameWithDefault)) ?~> "addLayer.nameInUse" _ <- bool2Fox( !annotationWithTracings.annotation.layers.exists( - _.`type` == AnnotationLayerTypeProto.skeleton && action.layerParameters.typ == AnnotationLayerType.Skeleton)) ?~> "addLayer.onlyOneSkeletonAllowed" + _.`type` == AnnotationLayerTypeProto.Skeleton && action.layerParameters.typ == AnnotationLayerType.Skeleton)) ?~> "addLayer.onlyOneSkeletonAllowed" tracing <- remoteWebknossosClient.createTracingFor(annotationId, action.layerParameters, previousVersion = targetVersion - 1) @@ -366,7 +366,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss requestAll: Boolean)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = { val skeletonTracingIds = if (requestAll) - annotation.layers.filter(_.`type` == AnnotationLayerTypeProto.skeleton).map(_.tracingId) + annotation.layers.filter(_.`type` == AnnotationLayerTypeProto.Skeleton).map(_.tracingId) else { (updates.flatMap { case u: SkeletonUpdateAction => Some(u.actionTracingId) @@ -375,7 +375,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } val volumeTracingIds = if (requestAll) - annotation.layers.filter(_.`type` == AnnotationLayerTypeProto.volume).map(_.tracingId) + annotation.layers.filter(_.`type` == AnnotationLayerTypeProto.Volume).map(_.tracingId) else { (updates.flatMap { case u: VolumeUpdateAction => Some(u.actionTracingId) @@ -670,7 +670,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss magRestrictions: MagRestrictions)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationLayerProto] = for { newTracingId <- layer.`type` match { - case AnnotationLayerTypeProto.volume => + case AnnotationLayerTypeProto.Volume => duplicateVolumeTracing(annotationId, layer.tracingId, version, @@ -679,7 +679,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss magRestrictions, editPosition, editRotation) - case AnnotationLayerTypeProto.skeleton => + case AnnotationLayerTypeProto.Skeleton => duplicateSkeletonTracing(annotationId, layer.tracingId, version, From 05b7e4075d17285813454940127844b8e9dcfa0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Mon, 28 Oct 2024 19:01:53 +0100 Subject: [PATCH 130/150] ensure loading newest annotation version from tracing store and patching annotation info with that information when loading an annotation --- frontend/javascripts/admin/admin_rest_api.ts | 20 +++++++---- .../oxalis/model/helpers/proto_helpers.ts | 16 ++++++++- .../oxalis/model_initialization.ts | 33 ++++++++++++++++--- frontend/javascripts/types/api_flow_types.ts | 13 ++++++++ 4 files changed, 71 insertions(+), 11 deletions(-) diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts index f02cc6946d9..5fec4b98d60 100644 --- a/frontend/javascripts/admin/admin_rest_api.ts +++ b/frontend/javascripts/admin/admin_rest_api.ts @@ -65,6 +65,7 @@ import { type VoxelSize, type APITimeTrackingPerUser, AnnotationLayerType, + type APITracingStoreAnnotation, } from "types/api_flow_types"; import { APIAnnotationTypeEnum } from "types/api_flow_types"; import type { LOG_LEVELS, Vector2, Vector3 } from "oxalis/constants"; @@ -87,6 +88,7 @@ import { enforceValidatedDatasetViewConfiguration } from "types/schemas/dataset_ import { parseProtoListOfLong, parseProtoTracing, + parseProtoTracingStoreAnnotation, serializeProtoListOfLong, } from "oxalis/model/helpers/proto_helpers"; import type { RequestOptions } from "libs/request"; @@ -913,15 +915,21 @@ export function getUpdateActionLog( }); } -export function getNewestVersionForTracing( +export async function getNewestVersionOfTracing( tracingStoreUrl: string, annotationId: string, -): Promise { - return doWithToken((token) => - Request.receiveJSON( - `${tracingStoreUrl}/tracings/annotation/${annotationId}/newestVersion?token=${token}`, - ).then((obj) => obj.version), +): Promise { + const annotationArrayBuffer = await doWithToken((token) => + Request.receiveArraybuffer( + `${tracingStoreUrl}/tracings/annotation/${annotationId}?token=${token}`, + { + headers: { + Accept: "application/x-protobuf", + }, + }, + ), ); + return parseProtoTracingStoreAnnotation(annotationArrayBuffer); } export function hasSegmentIndexInDataStore( diff --git a/frontend/javascripts/oxalis/model/helpers/proto_helpers.ts b/frontend/javascripts/oxalis/model/helpers/proto_helpers.ts index cd3430779d9..3af4f4e3c13 100644 --- a/frontend/javascripts/oxalis/model/helpers/proto_helpers.ts +++ b/frontend/javascripts/oxalis/model/helpers/proto_helpers.ts @@ -1,9 +1,11 @@ import { Root } from "protobufjs/light"; -import type { ServerTracing } from "types/api_flow_types"; +import type { APITracingStoreAnnotation, ServerTracing } from "types/api_flow_types"; // @ts-expect-error ts-migrate(2307) FIXME: Cannot find module 'SkeletonTracing.proto' or its ... Remove this comment to see the full error message import SkeletonTracingProto from "SkeletonTracing.proto"; // @ts-expect-error ts-migrate(2307) FIXME: Cannot find module 'VolumeTracing.proto' or its co... Remove this comment to see the full error message import VolumeTracingProto from "VolumeTracing.proto"; +// @ts-expect-error ts-migrate(2307) FIXME: Cannot find module 'AnnotationProto.proto' or its co... Remove this comment to see the full error message +import AnnotationProto from "Annotation.proto"; // @ts-expect-error ts-migrate(2307) FIXME: Cannot find module 'ListOfLong.proto' or its co... Remove this comment to see the full error message import ListOfLongProto from "ListOfLong.proto"; import { isBigInt } from "libs/utils"; @@ -64,4 +66,16 @@ export function parseProtoListOfLong( longs: Number, }).items; } + +export function parseProtoTracingStoreAnnotation(annotationArrayBuffer: ArrayBuffer): any { + const protoRoot = Root.fromJSON(AnnotationProto); + const messageType = protoRoot.lookupType(`${PROTO_PACKAGE}.AnnotationProto`); + const message = messageType.decode(new Uint8Array(annotationArrayBuffer)); + return messageType.toObject(message, { + arrays: true, + objects: true, + enums: String, + longs: Number, + }) as APITracingStoreAnnotation; +} export default {}; diff --git a/frontend/javascripts/oxalis/model_initialization.ts b/frontend/javascripts/oxalis/model_initialization.ts index c456701fbff..e753c152c7e 100644 --- a/frontend/javascripts/oxalis/model_initialization.ts +++ b/frontend/javascripts/oxalis/model_initialization.ts @@ -43,6 +43,7 @@ import { getDatasetViewConfiguration, getEditableMappingInfo, getAnnotationCompoundInformation, + getNewestVersionOfTracing, } from "admin/admin_rest_api"; import { dispatchMaybeFetchMeshFilesAsync, @@ -129,10 +130,34 @@ export async function initialize( if (initialCommandType.type === ControlModeEnum.TRACE) { const { annotationId } = initialCommandType; - annotation = - initialMaybeCompoundType != null - ? await getAnnotationCompoundInformation(annotationId, initialMaybeCompoundType) - : await getMaybeOutdatedAnnotationInformation(annotationId); + if (initialMaybeCompoundType != null) { + annotation = await getAnnotationCompoundInformation(annotationId, initialMaybeCompoundType); + } else { + let maybeOutdatedAnnotation = await getMaybeOutdatedAnnotationInformation(annotationId); + const annotationFromTracingStore = await getNewestVersionOfTracing( + maybeOutdatedAnnotation.tracingStore.url, + maybeOutdatedAnnotation.id, + ); + const completeAnnotation = { + ...maybeOutdatedAnnotation, + name: annotationFromTracingStore.name, + description: annotationFromTracingStore.description, + }; + annotationFromTracingStore.layers.forEach((layer) => { + if ( + maybeOutdatedAnnotation.annotationLayers.find((l) => l.tracingId === layer.tracingId) == + null + ) { + completeAnnotation.annotationLayers.push({ + tracingId: layer.tracingId, + name: layer.name, + typ: layer.type, + stats: {}, + }); + } + }); + annotation = completeAnnotation; + } datasetId = { name: annotation.dataSetName, owningOrganization: annotation.organization, diff --git a/frontend/javascripts/types/api_flow_types.ts b/frontend/javascripts/types/api_flow_types.ts index 42fcf83ce52..ee1a3d92706 100644 --- a/frontend/javascripts/types/api_flow_types.ts +++ b/frontend/javascripts/types/api_flow_types.ts @@ -575,6 +575,19 @@ export type APITimeTrackingPerAnnotation = { timeMillis: number; annotationLayerStats: Array; }; +type APITracingStoreAnnotationLayer = { + tracingId: string; + name: string; + type: AnnotationLayerType; +}; + +export type APITracingStoreAnnotation = { + name: string; + description: string; + version: number; + layers: APITracingStoreAnnotationLayer[]; +}; + export type APITimeTrackingPerUser = { user: APIUserCompact & { email: string; From 8d5be054e30e765dfabcbf056aefcb01b9e14240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Mon, 28 Oct 2024 19:29:12 +0100 Subject: [PATCH 131/150] send update annotation name and description as update actions --- frontend/javascripts/admin/admin_rest_api.ts | 2 -- .../model/actions/annotation_actions.ts | 4 +-- .../oxalis/model/sagas/annotation_saga.tsx | 33 ++++++++++++++++--- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts index 5fec4b98d60..b20e7978b7e 100644 --- a/frontend/javascripts/admin/admin_rest_api.ts +++ b/frontend/javascripts/admin/admin_rest_api.ts @@ -598,8 +598,6 @@ export function reOpenAnnotation( } export type EditableAnnotation = { - name: string; - description: string; visibility: APIAnnotationVisibility; tags: Array; viewConfiguration?: AnnotationViewConfiguration; diff --git a/frontend/javascripts/oxalis/model/actions/annotation_actions.ts b/frontend/javascripts/oxalis/model/actions/annotation_actions.ts index 1aa7ff5e470..a362151b945 100644 --- a/frontend/javascripts/oxalis/model/actions/annotation_actions.ts +++ b/frontend/javascripts/oxalis/model/actions/annotation_actions.ts @@ -20,10 +20,10 @@ import Deferred from "libs/async/deferred"; import type { AdditionalCoordinate } from "types/api_flow_types"; type InitializeAnnotationAction = ReturnType; -type SetAnnotationNameAction = ReturnType; +export type SetAnnotationNameAction = ReturnType; type SetAnnotationVisibilityAction = ReturnType; export type EditAnnotationLayerAction = ReturnType; -type SetAnnotationDescriptionAction = ReturnType; +export type SetAnnotationDescriptionAction = ReturnType; type SetAnnotationAllowUpdateAction = ReturnType; type SetBlockedByUserAction = ReturnType; type SetUserBoundingBoxesAction = ReturnType; diff --git a/frontend/javascripts/oxalis/model/sagas/annotation_saga.tsx b/frontend/javascripts/oxalis/model/sagas/annotation_saga.tsx index 2b036703d5c..5bb550e9eb0 100644 --- a/frontend/javascripts/oxalis/model/sagas/annotation_saga.tsx +++ b/frontend/javascripts/oxalis/model/sagas/annotation_saga.tsx @@ -4,6 +4,8 @@ import type { Action } from "oxalis/model/actions/actions"; import { type EditAnnotationLayerAction, setAnnotationAllowUpdateAction, + type SetAnnotationDescriptionAction, + type SetAnnotationNameAction, setBlockedByUserAction, type SetOthersMayEditForAnnotationAction, } from "oxalis/model/actions/annotation_actions"; @@ -44,13 +46,36 @@ import { getLastActiveLayout, getLayoutConfig } from "oxalis/view/layouting/layo import { is3dViewportMaximized } from "oxalis/view/layouting/flex_layout_helper"; import { needsLocalHdf5Mapping } from "../accessors/volumetracing_accessor"; import { pushSaveQueueTransaction } from "../actions/save_actions"; -import { updateAnnotationLayerName } from "./update_actions"; +import { updateAnnotationLayerName, updateMetadataOfAnnotation } from "./update_actions"; /* Note that this must stay in sync with the back-end constant MaxMagForAgglomerateMapping compare https://github.com/scalableminds/webknossos/issues/5223. */ const MAX_MAG_FOR_AGGLOMERATE_MAPPING = 16; +export function* pushAnnotationNameUpdateAction(action: SetAnnotationNameAction) { + const mayEdit = yield* select((state) => mayEditAnnotationProperties(state)); + if (!mayEdit) { + return; + } + yield* put( + pushSaveQueueTransaction([updateMetadataOfAnnotation(action.name)], "unused-tracing-id"), + ); +} + +export function* pushAnnotationDescriptionUpdateAction(action: SetAnnotationDescriptionAction) { + const mayEdit = yield* select((state) => mayEditAnnotationProperties(state)); + if (!mayEdit) { + return; + } + yield* put( + pushSaveQueueTransaction( + [updateMetadataOfAnnotation(undefined, action.description)], + "unused-tracing-id", + ), + ); +} + export function* pushAnnotationUpdateAsync(action: Action) { const tracing = yield* select((state) => state.tracing); const mayEdit = yield* select((state) => mayEditAnnotationProperties(state)); @@ -68,9 +93,7 @@ export function* pushAnnotationUpdateAsync(action: Action) { }; // The extra type annotation is needed here for flow const editObject: Partial = { - name: tracing.name, visibility: tracing.visibility, - description: tracing.description, viewConfiguration, }; try { @@ -207,9 +230,9 @@ export function* watchAnnotationAsync(): Saga { // name, only the latest action is relevant. If `_takeEvery` was used, // all updates to the annotation name would be retried regularly, which // would also cause race conditions. - yield* takeLatest("SET_ANNOTATION_NAME", pushAnnotationUpdateAsync); + yield* takeLatest("SET_ANNOTATION_NAME", pushAnnotationNameUpdateAction); yield* takeLatest("SET_ANNOTATION_VISIBILITY", pushAnnotationUpdateAsync); - yield* takeLatest("SET_ANNOTATION_DESCRIPTION", pushAnnotationUpdateAsync); + yield* takeLatest("SET_ANNOTATION_DESCRIPTION", pushAnnotationDescriptionUpdateAction); yield* takeLatest( ((action: Action) => action.type === "UPDATE_LAYER_SETTING" && From 85dece0ef3a0661bb8ee338154ecd6ce62ba3f95 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 29 Oct 2024 09:24:03 +0100 Subject: [PATCH 132/150] WIP task creation --- app/controllers/AnnotationController.scala | 3 +- app/models/annotation/AnnotationService.scala | 1 + .../WKRemoteTracingStoreClient.scala | 33 ++++++++++++++----- app/models/task/TaskCreationService.scala | 7 +++- .../webknossos/datastore/rpc/RPCRequest.scala | 2 +- .../annotation/TSAnnotationService.scala | 6 ++++ .../controllers/TSAnnotationController.scala | 5 ++- .../volume/VolumeTracingService.scala | 31 +++++++++-------- ...alableminds.webknossos.tracingstore.routes | 2 +- 9 files changed, 63 insertions(+), 27 deletions(-) diff --git a/app/controllers/AnnotationController.scala b/app/controllers/AnnotationController.scala index 9a915500a90..689586935b4 100755 --- a/app/controllers/AnnotationController.scala +++ b/app/controllers/AnnotationController.scala @@ -441,7 +441,8 @@ class AnnotationController @Inject()( isFromTask = annotation._task.isDefined, editPosition = None, editRotation = None, - boundingBox = dataSource.map(_.boundingBox), + boundingBox = None, + datasetBoundingBox = dataSource.map(_.boundingBox), magRestrictions = MagRestrictions.empty ) newAnnotationLayers = newAnnotationProto.layers.map(AnnotationLayer.fromProto) diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index bade14cfdf5..93d9269aaef 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -372,6 +372,7 @@ class AnnotationService @Inject()( editPosition = None, editRotation = None, boundingBox = None, + datasetBoundingBox = None, magRestrictions = MagRestrictions.empty ) newAnnotation = annotationBase.copy( diff --git a/app/models/annotation/WKRemoteTracingStoreClient.scala b/app/models/annotation/WKRemoteTracingStoreClient.scala index b66ccee76b6..e5e2bb04eda 100644 --- a/app/models/annotation/WKRemoteTracingStoreClient.scala +++ b/app/models/annotation/WKRemoteTracingStoreClient.scala @@ -90,6 +90,8 @@ class WKRemoteTracingStoreClient( .postProto[AnnotationProto](annotationProto) } + // Used in duplicate route. History and version are kept + // TODO: can we remove some params here, if they are used only in task case? def duplicateAnnotation(annotationId: ObjectId, newAnnotationId: ObjectId, version: Option[Long], @@ -97,6 +99,7 @@ class WKRemoteTracingStoreClient( editPosition: Option[Vec3Int], editRotation: Option[Vec3Double], boundingBox: Option[BoundingBox], + datasetBoundingBox: Option[BoundingBox], magRestrictions: MagRestrictions, ): Fox[AnnotationProto] = { logger.debug(s"Called to duplicate annotation $annotationId." + baseInfo) @@ -107,27 +110,41 @@ class WKRemoteTracingStoreClient( .addQueryStringOptional("editPosition", editPosition.map(_.toUriLiteral)) .addQueryStringOptional("editRotation", editRotation.map(_.toUriLiteral)) .addQueryStringOptional("boundingBox", boundingBox.map(_.toLiteral)) + .addQueryStringOptional("datasetBoundingBox", datasetBoundingBox.map(_.toLiteral)) .addQueryString("isFromTask" -> isFromTask.toString) .addQueryStringOptional("minMag", magRestrictions.minStr) .addQueryStringOptional("maxMag", magRestrictions.maxStr) .postWithProtoResponse[AnnotationProto]()(AnnotationProto) } - def duplicateSkeletonTracing(skeletonTracingId: String, - versionString: Option[String] = None, - isFromTask: Boolean = false, + // Used in task creation. History is dropped, new version will be zero. + // TODO: currently also used in resetToBase. Fix that. + def duplicateSkeletonTracing(skeletonTracingId: String, // TODO: might also need annotation id editPosition: Option[Vec3Int] = None, editRotation: Option[Vec3Double] = None, - boundingBox: Option[BoundingBox] = None): Fox[String] = ??? + boundingBox: Option[BoundingBox] = None): Fox[String] = + rpc(s"${tracingStore.url}/tracings/skeleton/$skeletonTracingId/duplicate").withLongTimeout + .addQueryString("token" -> RpcTokenHolder.webknossosToken) + .addQueryStringOptional("editPosition", editPosition.map(_.toUriLiteral)) + .addQueryStringOptional("editRotation", editRotation.map(_.toUriLiteral)) + .addQueryStringOptional("boundingBox", boundingBox.map(_.toLiteral)) + .postWithJsonResponse[String]() + // Used in task creation. History is dropped, new version will be zero. + // TODO: currently also used in resetToBase. Fix that. def duplicateVolumeTracing(volumeTracingId: String, - isFromTask: Boolean = false, - datasetBoundingBox: Option[BoundingBox] = None, magRestrictions: MagRestrictions = MagRestrictions.empty, - downsample: Boolean = false, editPosition: Option[Vec3Int] = None, editRotation: Option[Vec3Double] = None, - boundingBox: Option[BoundingBox] = None): Fox[String] = ??? + boundingBox: Option[BoundingBox] = None): Fox[String] = + rpc(s"${tracingStore.url}/tracings/volume/$volumeTracingId/duplicate").withLongTimeout + .addQueryString("token" -> RpcTokenHolder.webknossosToken) + .addQueryStringOptional("editPosition", editPosition.map(_.toUriLiteral)) + .addQueryStringOptional("editRotation", editRotation.map(_.toUriLiteral)) + .addQueryStringOptional("boundingBox", boundingBox.map(_.toLiteral)) + .addQueryStringOptional("minMag", magRestrictions.minStr) + .addQueryStringOptional("maxMag", magRestrictions.maxStr) + .postWithJsonResponse[String]() def mergeSkeletonTracingsByIds(tracingIds: List[String], persistTracing: Boolean): Fox[String] = { logger.debug("Called to merge SkeletonTracings by ids." + baseInfo) diff --git a/app/models/task/TaskCreationService.scala b/app/models/task/TaskCreationService.scala index 865316eef8b..cd79b8e3287 100644 --- a/app/models/task/TaskCreationService.scala +++ b/app/models/task/TaskCreationService.scala @@ -150,7 +150,12 @@ class TaskCreationService @Inject()(taskTypeService: TaskTypeService, for { volumeTracingOpt <- baseAnnotation.volumeTracingId newVolumeTracingId <- volumeTracingOpt - .map(id => tracingStoreClient.duplicateVolumeTracing(id, magRestrictions = magRestrictions)) + .map( + id => + tracingStoreClient.duplicateVolumeTracing(id, + editPosition = Some(params.editPosition), + editRotation = Some(params.editRotation), + magRestrictions = magRestrictions)) .getOrElse( annotationService .createVolumeTracingBase( diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala index c0d9b8695d2..3830553d25b 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/rpc/RPCRequest.scala @@ -114,7 +114,7 @@ class RPCRequest(val id: Int, val url: String, wsClient: WSClient)(implicit ec: parseJsonResponse(performRequest) } - def postWithJsonResponse[T: Reads]: Fox[T] = { + def postWithJsonResponse[T: Reads](): Fox[T] = { request = request.withMethod("POST") parseJsonResponse(performRequest) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 627e4b640c7..914be0e07ff 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -629,6 +629,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss editPosition: Option[Vec3Int], editRotation: Option[Vec3Double], boundingBox: Option[BoundingBox], + datasetBoundingBox: Option[BoundingBox], magRestrictions: MagRestrictions)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationProto] = for { currentAnnotation <- get(annotationId, version) @@ -641,6 +642,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss editPosition, editRotation, boundingBox, + datasetBoundingBox, magRestrictions)) _ <- duplicateUpdates(annotationId, newAnnotationId) duplicatedAnnotation = currentAnnotation.copy(layers = newLayers) @@ -667,6 +669,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss editPosition: Option[Vec3Int], editRotation: Option[Vec3Double], boundingBox: Option[BoundingBox], + datasetBoundingBox: Option[BoundingBox], magRestrictions: MagRestrictions)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationLayerProto] = for { newTracingId <- layer.`type` match { @@ -676,6 +679,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss version, isFromTask, boundingBox, + datasetBoundingBox, magRestrictions, editPosition, editRotation) @@ -697,6 +701,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss version: Long, isFromTask: Boolean, boundingBox: Option[BoundingBox], + datasetBoundingBox: Option[BoundingBox], magRestrictions: MagRestrictions, editPosition: Option[Vec3Int], editRotation: Option[Vec3Double])(implicit ec: ExecutionContext, tc: TokenContext): Fox[String] = { @@ -708,6 +713,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss sourceTracing, isFromTask, boundingBox, + datasetBoundingBox, magRestrictions, editPosition, editRotation, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index 2058d2d6e23..cbc5d219aa9 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -109,7 +109,8 @@ class TSAnnotationController @Inject()( maxMag: Option[Int], editPosition: Option[String], editRotation: Option[String], - boundingBox: Option[String]): Action[AnyContent] = + boundingBox: Option[String], + datasetBoundingBox: Option[String]): Action[AnyContent] = Action.async { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { @@ -118,6 +119,7 @@ class TSAnnotationController @Inject()( editPositionParsed <- Fox.runOptional(editPosition)(Vec3Int.fromUriLiteral) editRotationParsed <- Fox.runOptional(editRotation)(Vec3Double.fromUriLiteral) boundingBoxParsed <- Fox.runOptional(boundingBox)(BoundingBox.fromLiteral) + datasetBoundingBoxParsed <- Fox.runOptional(datasetBoundingBox)(BoundingBox.fromLiteral) magRestrictions = MagRestrictions(minMag, maxMag) annotationProto <- annotationService.duplicate(annotationId, newAnnotationId, @@ -126,6 +128,7 @@ class TSAnnotationController @Inject()( editPositionParsed, editRotationParsed, boundingBoxParsed, + datasetBoundingBoxParsed, magRestrictions) } yield Ok(annotationProto.toByteArray).as(protobufMimeType) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 291fd749f39..39523d44a08 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -491,6 +491,7 @@ class VolumeTracingService @Inject()( sourceTracing: VolumeTracing, isFromTask: Boolean, boundingBox: Option[BoundingBox], + datasetBoundingBox: Option[BoundingBox], magRestrictions: MagRestrictions, editPosition: Option[Vec3Int], editRotation: Option[Vec3Double], @@ -516,21 +517,23 @@ class VolumeTracingService @Inject()( } yield newTracing } - @SuppressWarnings(Array("OptionGet")) //We suppress this warning because we check the option beforehand private def addBoundingBoxFromTaskIfRequired(tracing: VolumeTracing, - fromTask: Boolean, - datasetBoundingBox: Option[BoundingBox]): VolumeTracing = - if (fromTask && datasetBoundingBox.isDefined) { - val newId = if (tracing.userBoundingBoxes.isEmpty) 1 else tracing.userBoundingBoxes.map(_.id).max + 1 - tracing - .addUserBoundingBoxes( - NamedBoundingBoxProto(newId, - Some("task bounding box"), - Some(true), - Some(getRandomColor), - tracing.boundingBox)) - .withBoundingBox(datasetBoundingBox.get) - } else tracing + isFromTask: Boolean, + datasetBoundingBoxOpt: Option[BoundingBox]): VolumeTracing = + datasetBoundingBoxOpt match { + case Some(datasetBoundingBox) if isFromTask => { + val newId = if (tracing.userBoundingBoxes.isEmpty) 1 else tracing.userBoundingBoxes.map(_.id).max + 1 + tracing + .addUserBoundingBoxes( + NamedBoundingBoxProto(newId, + Some("task bounding box"), + Some(true), + Some(getRandomColor), + tracing.boundingBox)) + .withBoundingBox(datasetBoundingBox) + } + case _ => tracing + } def duplicateVolumeData(sourceTracingId: String, sourceTracing: VolumeTracing, diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index e5c6ec7a9a1..9f5f3337c2a 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -11,7 +11,7 @@ POST /annotation/:annotationId/update GET /annotation/:annotationId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionLog(annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]) GET /annotation/:annotationId/updateActionStatistics @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionStatistics(annotationId: String) GET /annotation/:annotationId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.newestVersion(annotationId: String) -POST /annotation/:annotationId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.duplicate(annotationId: String, newAnnotationId: String, version: Option[Long], isFromTask: Boolean, minMag: Option[Int], maxMag: Option[Int], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) +POST /annotation/:annotationId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.duplicate(annotationId: String, newAnnotationId: String, version: Option[Long], isFromTask: Boolean, minMag: Option[Int], maxMag: Option[Int], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String], datasetBoundingBox: Option[String]) # Volume tracings POST /volume/save @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.save() From a543415fe3bca347bf62bbc22dc16a57bb282988 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 29 Oct 2024 09:56:23 +0100 Subject: [PATCH 133/150] fix task creation without base --- app/models/annotation/AnnotationService.scala | 14 ++++++++++++-- .../annotation/WKRemoteTracingStoreClient.scala | 3 ++- app/models/task/TaskCreationService.scala | 5 +++-- .../models/annotation/AnnotationLayer.scala | 5 ++++- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index 93d9269aaef..d821d478068 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -364,6 +364,8 @@ class AnnotationService @Inject()( dataset <- datasetDAO.findOne(annotationBase._dataset) ?~> Messages("dataset.noAccess", datasetName) _ <- bool2Fox(dataset.isUsable) ?~> Messages("dataset.notImported", dataset.name) tracingStoreClient <- tracingStoreService.clientFor(dataset) + _ = logger.info( + f"task assignment. creating annotation $initializingAnnotationId from base $annotationBaseId for task $taskId") duplicatedAnnotationProto <- tracingStoreClient.duplicateAnnotation( annotationBaseId, initializingAnnotationId, @@ -458,13 +460,15 @@ class AnnotationService @Inject()( case _ => annotationDAO.abortInitializingAnnotation(initializingAnnotationId) } - def createAnnotationBase( + // Save annotation base to postgres AND annotation proto to tracingstore. + def createAndSaveAnnotationBase( taskFox: Fox[Task], userId: ObjectId, skeletonTracingIdBox: Box[Option[String]], volumeTracingIdBox: Box[Option[String]], datasetId: ObjectId, - description: Option[String] + description: Option[String], + tracingStoreClient: WKRemoteTracingStoreClient )(implicit ctx: DBAccessContext): Fox[Unit] = for { task <- taskFox @@ -481,6 +485,12 @@ class AnnotationService @Inject()( annotationLayers, description.getOrElse(""), typ = AnnotationType.TracingBase) + annotationBaseProto = AnnotationProto(name = Some(AnnotationDefaults.defaultName), + description = Some(AnnotationDefaults.defaultDescription), + version = 0L, + layers = annotationLayers.map(_.toProto)) + _ <- tracingStoreClient.saveAnnotationProto(annotationBase._id, annotationBaseProto) + _ = logger.info(s"inserting base annotation ${annotationBase._id} for task ${task._id}") _ <- annotationDAO.insertOne(annotationBase) } yield () diff --git a/app/models/annotation/WKRemoteTracingStoreClient.scala b/app/models/annotation/WKRemoteTracingStoreClient.scala index e5e2bb04eda..6f072bbfc8d 100644 --- a/app/models/annotation/WKRemoteTracingStoreClient.scala +++ b/app/models/annotation/WKRemoteTracingStoreClient.scala @@ -83,7 +83,8 @@ class WKRemoteTracingStoreClient( } def saveAnnotationProto(annotationId: ObjectId, annotationProto: AnnotationProto): Fox[Unit] = { - logger.debug("Called to save AnnotationProto." + baseInfo) + logger.debug( + f"Called to save AnnotationProto $annotationId with layers ${annotationProto.layers.map(_.tracingId).mkString(",")}." + baseInfo) rpc(s"${tracingStore.url}/tracings/annotation/save") .addQueryString("token" -> RpcTokenHolder.webknossosToken) .addQueryString("annotationId" -> annotationId.toString) diff --git a/app/models/task/TaskCreationService.scala b/app/models/task/TaskCreationService.scala index cd79b8e3287..3fbcb651bf7 100644 --- a/app/models/task/TaskCreationService.scala +++ b/app/models/task/TaskCreationService.scala @@ -428,13 +428,14 @@ class TaskCreationService @Inject()(taskTypeService: TaskTypeService, .toList createAnnotationBaseResults: List[Fox[Unit]] = zipped.map( tuple => - annotationService.createAnnotationBase( + annotationService.createAndSaveAnnotationBase( taskFox = tuple._3, requestingUser._id, skeletonTracingIdBox = tuple._2._1, volumeTracingIdBox = tuple._2._2, dataset._id, - description = tuple._1.map(_._1.description).openOr(None) + description = tuple._1.map(_._1.description).openOr(None), + tracingStoreClient )) warnings <- warnIfTeamHasNoAccess(fullTasks.map(_._1), dataset, requestingUser) zippedTasksAndAnnotations = taskObjects zip createAnnotationBaseResults diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayer.scala index 5b1ba9b6607..b9552941679 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayer.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/annotation/AnnotationLayer.scala @@ -17,7 +17,10 @@ case class AnnotationLayer( typ: AnnotationLayerType, name: String, stats: JsObject, -) +) { + def toProto: AnnotationLayerProto = + AnnotationLayerProto(tracingId, name, AnnotationLayerType.toProto(typ)) +} object AnnotationLayer extends FoxImplicits { implicit val jsonFormat: OFormat[AnnotationLayer] = Json.format[AnnotationLayer] From d8d7fd4080b75e745f2bfb0af91eef9d28f21c0c Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 29 Oct 2024 10:25:15 +0100 Subject: [PATCH 134/150] wip task creation from base id --- .../annotation/TSAnnotationService.scala | 70 +++++++++++-------- .../SkeletonTracingController.scala | 31 ++++++++ .../controllers/TSAnnotationController.scala | 2 +- .../controllers/VolumeTracingController.scala | 35 ++++++++++ .../EditableMappingService.scala | 30 +++++--- ...alableminds.webknossos.tracingstore.routes | 2 + 6 files changed, 128 insertions(+), 42 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 914be0e07ff..04206a50b8f 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -677,6 +677,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss duplicateVolumeTracing(annotationId, layer.tracingId, version, + version, isFromTask, boundingBox, datasetBoundingBox, @@ -687,6 +688,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss duplicateSkeletonTracing(annotationId, layer.tracingId, version, + version, isFromTask, editPosition, editRotation, @@ -695,19 +697,20 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } } yield layer.copy(tracingId = newTracingId) - private def duplicateVolumeTracing( - annotationId: String, - sourceTracingId: String, - version: Long, - isFromTask: Boolean, - boundingBox: Option[BoundingBox], - datasetBoundingBox: Option[BoundingBox], - magRestrictions: MagRestrictions, - editPosition: Option[Vec3Int], - editRotation: Option[Vec3Double])(implicit ec: ExecutionContext, tc: TokenContext): Fox[String] = { + def duplicateVolumeTracing( + sourceAnnotationId: String, + sourceTracingId: String, + sourceVersion: Long, + newVersion: Long, + isFromTask: Boolean, + boundingBox: Option[BoundingBox], + datasetBoundingBox: Option[BoundingBox], + magRestrictions: MagRestrictions, + editPosition: Option[Vec3Int], + editRotation: Option[Vec3Double])(implicit ec: ExecutionContext, tc: TokenContext): Fox[String] = { val newTracingId = TracingId.generate for { - sourceTracing <- findVolume(annotationId, sourceTracingId, Some(version)) + sourceTracing <- findVolume(sourceAnnotationId, sourceTracingId, Some(sourceVersion)) newTracing <- volumeTracingService.adaptVolumeForDuplicate(sourceTracingId, newTracingId, sourceTracing, @@ -717,44 +720,49 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss magRestrictions, editPosition, editRotation, - version) - _ <- tracingDataStore.volumes.put(newTracingId, version, newTracing) + newVersion) + _ <- tracingDataStore.volumes.put(newTracingId, newVersion, newTracing) _ <- Fox.runIf(!newTracing.getHasEditableMapping)( volumeTracingService.duplicateVolumeData(sourceTracingId, sourceTracing, newTracingId, newTracing)) _ <- Fox.runIf(newTracing.getHasEditableMapping)( - duplicateEditableMapping(annotationId, sourceTracingId, newTracingId, version)) + duplicateEditableMapping(sourceAnnotationId, sourceTracingId, newTracingId, sourceVersion, newVersion)) } yield newTracingId } - private def duplicateEditableMapping(annotationId: String, + private def duplicateEditableMapping(sourceAnnotationId: String, sourceTracingId: String, newTracingId: String, - version: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Unit] = + sourceVersion: Long, + newVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Unit] = for { - editableMappingInfo <- findEditableMappingInfo(annotationId, sourceTracingId, Some(version)) - _ <- tracingDataStore.editableMappingsInfo.put(newTracingId, version, toProtoBytes(editableMappingInfo)) - _ <- editableMappingService.duplicateSegmentToAgglomerate(sourceTracingId, newTracingId, version) - _ <- editableMappingService.duplicateAgglomerateToGraph(sourceTracingId, newTracingId, version) + editableMappingInfo <- findEditableMappingInfo(sourceAnnotationId, sourceTracingId, Some(sourceVersion)) + _ <- tracingDataStore.editableMappingsInfo.put(newTracingId, newVersion, toProtoBytes(editableMappingInfo)) + _ <- editableMappingService.duplicateSegmentToAgglomerate(sourceTracingId, + newTracingId, + sourceVersion, + newVersion) + _ <- editableMappingService.duplicateAgglomerateToGraph(sourceTracingId, newTracingId, sourceVersion, newVersion) } yield () - private def duplicateSkeletonTracing( - annotationId: String, - tracingId: String, - version: Long, - isFromTask: Boolean, - editPosition: Option[Vec3Int], - editRotation: Option[Vec3Double], - boundingBox: Option[BoundingBox])(implicit ec: ExecutionContext, tc: TokenContext): Fox[String] = { + def duplicateSkeletonTracing( + sourceAnnotationId: String, + sourceTracingId: String, + sourceVersion: Long, + newVersion: Long, + isFromTask: Boolean, + editPosition: Option[Vec3Int], + editRotation: Option[Vec3Double], + boundingBox: Option[BoundingBox])(implicit ec: ExecutionContext, tc: TokenContext): Fox[String] = { val newTracingId = TracingId.generate for { - skeleton <- findSkeleton(annotationId, tracingId, Some(version)) + skeleton <- findSkeleton(sourceAnnotationId, sourceTracingId, Some(sourceVersion)) adaptedSkeleton = skeletonTracingService.adaptSkeletonForDuplicate(skeleton, isFromTask, editPosition, editRotation, boundingBox, - version) - _ <- tracingDataStore.skeletons.put(newTracingId, version, adaptedSkeleton) + newVersion) + _ <- tracingDataStore.skeletons.put(newTracingId, newVersion, adaptedSkeleton) } yield newTracingId } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala index bcfe15941fd..40fca232591 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala @@ -1,6 +1,7 @@ package com.scalableminds.webknossos.tracingstore.controllers import com.google.inject.Inject +import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int} import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.SkeletonTracing.{SkeletonTracing, SkeletonTracingOpt, SkeletonTracings} import com.scalableminds.webknossos.datastore.services.UserAccessRequest @@ -142,4 +143,34 @@ class SkeletonTracingController @Inject()(skeletonTracingService: SkeletonTracin } } + // Used in task creation. History is dropped. Caller is responsible to create and save a matching AnnotationProto object + def duplicate(tracingId: String, + editPosition: Option[String], + editRotation: Option[String], + boundingBox: Option[String]): Action[AnyContent] = + Action.async { implicit request => + log() { + logTime(slackNotificationService.noticeSlowRequest) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { + for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) + editPositionParsed <- Fox.runOptional(editPosition)(Vec3Int.fromUriLiteral) + editRotationParsed <- Fox.runOptional(editRotation)(Vec3Double.fromUriLiteral) + boundingBoxParsed <- Fox.runOptional(boundingBox)(BoundingBox.fromLiteral) + newestSourceVersion <- annotationService.currentMaterializableVersion(annotationId) + newTracingId <- annotationService.duplicateSkeletonTracing( + annotationId, + sourceTracingId = tracingId, + sourceVersion = newestSourceVersion, + newVersion = 0, + editPosition = editPositionParsed, + editRotation = editRotationParsed, + boundingBox = boundingBoxParsed, + isFromTask = false + ) + } yield Ok(Json.toJson(newTracingId)) + } + } + } + } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index cbc5d219aa9..c27795c0cb4 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -114,7 +114,7 @@ class TSAnnotationController @Inject()( Action.async { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeAnnotation(annotationId)) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readAnnotation(annotationId)) { for { editPositionParsed <- Fox.runOptional(editPosition)(Vec3Int.fromUriLiteral) editRotationParsed <- Fox.runOptional(editRotation)(Vec3Double.fromUriLiteral) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index a56a441fba6..8c01ae36346 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -414,4 +414,39 @@ class VolumeTracingController @Inject()( } } + // Used in task creation. History is dropped. Caller is responsible to create and save a matching AnnotationProto object + def duplicate(tracingId: String, + minMag: Option[Int], + maxMag: Option[Int], + editPosition: Option[String], + editRotation: Option[String], + boundingBox: Option[String]): Action[AnyContent] = + Action.async { implicit request => + log() { + logTime(slackNotificationService.noticeSlowRequest) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readTracing(tracingId)) { + for { + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) + editPositionParsed <- Fox.runOptional(editPosition)(Vec3Int.fromUriLiteral) + editRotationParsed <- Fox.runOptional(editRotation)(Vec3Double.fromUriLiteral) + boundingBoxParsed <- Fox.runOptional(boundingBox)(BoundingBox.fromLiteral) + magRestrictions = MagRestrictions(minMag, maxMag) + newestSourceVersion <- annotationService.currentMaterializableVersion(annotationId) + newTracingId <- annotationService.duplicateVolumeTracing( + annotationId, + sourceTracingId = tracingId, + sourceVersion = newestSourceVersion, + newVersion = 0, + editPosition = editPositionParsed, + editRotation = editRotationParsed, + boundingBox = boundingBoxParsed, + datasetBoundingBox = None, + isFromTask = false, + magRestrictions = magRestrictions + ) + } yield Ok(Json.toJson(newTracingId)) + } + } + } + } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala index 66dd0e32eb7..53552cc74b1 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingService.scala @@ -135,31 +135,41 @@ class EditableMappingService @Inject()( } yield newEditableMappingInfo } - def duplicateSegmentToAgglomerate(sourceTracingId: String, newId: String, version: Long): Fox[Unit] = { - val iterator = + def duplicateSegmentToAgglomerate(sourceTracingId: String, + newId: String, + sourceVersion: Long, + newVersion: Long): Fox[Unit] = { + val sourceIterator = new VersionedFossilDbIterator(sourceTracingId, tracingDataStore.editableMappingsSegmentToAgglomerate, - Some(version)) + Some(sourceVersion)) for { - _ <- Fox.combined(iterator.map { keyValuePair => + _ <- Fox.combined(sourceIterator.map { keyValuePair => for { chunkId <- chunkIdFromSegmentToAgglomerateKey(keyValuePair.key).toFox newKey = segmentToAgglomerateKey(newId, chunkId) - _ <- tracingDataStore.editableMappingsSegmentToAgglomerate.put(newKey, version = version, keyValuePair.value) + _ <- tracingDataStore.editableMappingsSegmentToAgglomerate.put(newKey, + version = newVersion, + keyValuePair.value) } yield () }.toList) } yield () } - def duplicateAgglomerateToGraph(sourceTracingId: String, newId: String, version: Long): Fox[Unit] = { - val iterator = - new VersionedFossilDbIterator(sourceTracingId, tracingDataStore.editableMappingsAgglomerateToGraph, Some(version)) + def duplicateAgglomerateToGraph(sourceTracingId: String, + newId: String, + sourceVersion: Long, + newVersion: Long): Fox[Unit] = { + val sourceIterator = + new VersionedFossilDbIterator(sourceTracingId, + tracingDataStore.editableMappingsAgglomerateToGraph, + Some(sourceVersion)) for { - _ <- Fox.combined(iterator.map { keyValuePair => + _ <- Fox.combined(sourceIterator.map { keyValuePair => for { agglomerateId <- agglomerateIdFromAgglomerateGraphKey(keyValuePair.key).toFox newKey = agglomerateGraphKey(newId, agglomerateId) - _ <- tracingDataStore.editableMappingsAgglomerateToGraph.put(newKey, version = version, keyValuePair.value) + _ <- tracingDataStore.editableMappingsAgglomerateToGraph.put(newKey, version = newVersion, keyValuePair.value) } yield () }.toList) } yield () diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 9f5f3337c2a..06b9967a84c 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -22,6 +22,7 @@ GET /volume/:tracingId/allDataZip POST /volume/:tracingId/data @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.data(tracingId: String) POST /volume/:tracingId/adHocMesh @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.requestAdHocMesh(tracingId: String) POST /volume/:tracingId/fullMesh.stl @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.loadFullMeshStl(tracingId: String) +POST /volume/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.duplicate(tracingId: String, minMag: Option[Int], maxMag: Option[Int], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) POST /volume/:tracingId/segmentIndex/:segmentId @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentIndex(tracingId: String, segmentId: Long) POST /volume/:tracingId/importVolumeData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.importVolumeData(tracingId: String) GET /volume/:tracingId/findData @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.findData(tracingId: String) @@ -72,4 +73,5 @@ POST /skeleton/saveMultiple POST /skeleton/mergedFromContents @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.mergedFromContents(persist: Boolean) POST /skeleton/mergedFromIds @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.mergedFromIds(persist: Boolean) GET /skeleton/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.get(tracingId: String, version: Option[Long]) +POST /skeleton/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.duplicate(tracingId: String, editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) POST /skeleton/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.getMultiple From 41a326c5142067e4a440f6af6ca6558000b0a71a Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 29 Oct 2024 11:19:07 +0100 Subject: [PATCH 135/150] fix volume task creation with base id --- .../annotation/TSAnnotationService.scala | 48 +++++++++---------- .../controllers/VolumeTracingController.scala | 8 +--- .../tracings/TracingService.scala | 2 - .../volume/VolumeTracingService.scala | 28 ++++++----- 4 files changed, 40 insertions(+), 46 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 04206a50b8f..a449d6d80d0 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -564,14 +564,10 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss version: Option[Long] = None, useCache: Boolean = true, applyUpdates: Boolean = false)(implicit tc: TokenContext, ec: ExecutionContext): Fox[VolumeTracing] = - if (tracingId == TracingId.dummy) - Fox.successful(volumeTracingService.dummyTracing) - else { - for { - annotation <- getWithTracings(annotationId, version, List.empty, List(tracingId), requestAll = false) // TODO is applyUpdates still needed? - tracing <- annotation.getVolume(tracingId) - } yield tracing - } + for { + annotation <- getWithTracings(annotationId, version, List.empty, List(tracingId), requestAll = false) // TODO is applyUpdates still needed? + tracing <- annotation.getVolume(tracingId) + } yield tracing def findSkeleton( annotationId: String, @@ -698,16 +694,16 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } yield layer.copy(tracingId = newTracingId) def duplicateVolumeTracing( - sourceAnnotationId: String, - sourceTracingId: String, - sourceVersion: Long, - newVersion: Long, - isFromTask: Boolean, - boundingBox: Option[BoundingBox], - datasetBoundingBox: Option[BoundingBox], - magRestrictions: MagRestrictions, - editPosition: Option[Vec3Int], - editRotation: Option[Vec3Double])(implicit ec: ExecutionContext, tc: TokenContext): Fox[String] = { + sourceAnnotationId: String, + sourceTracingId: String, + sourceVersion: Long, + newVersion: Long, + isFromTask: Boolean, + boundingBox: Option[BoundingBox], + datasetBoundingBox: Option[BoundingBox], + magRestrictions: MagRestrictions, + editPosition: Option[Vec3Int], + editRotation: Option[Vec3Double])(implicit ec: ExecutionContext, tc: TokenContext): Fox[String] = { val newTracingId = TracingId.generate for { sourceTracing <- findVolume(sourceAnnotationId, sourceTracingId, Some(sourceVersion)) @@ -745,14 +741,14 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } yield () def duplicateSkeletonTracing( - sourceAnnotationId: String, - sourceTracingId: String, - sourceVersion: Long, - newVersion: Long, - isFromTask: Boolean, - editPosition: Option[Vec3Int], - editRotation: Option[Vec3Double], - boundingBox: Option[BoundingBox])(implicit ec: ExecutionContext, tc: TokenContext): Fox[String] = { + sourceAnnotationId: String, + sourceTracingId: String, + sourceVersion: Long, + newVersion: Long, + isFromTask: Boolean, + editPosition: Option[Vec3Int], + editRotation: Option[Vec3Double], + boundingBox: Option[BoundingBox])(implicit ec: ExecutionContext, tc: TokenContext): Fox[String] = { val newTracingId = TracingId.generate for { skeleton <- findSkeleton(sourceAnnotationId, sourceTracingId, Some(sourceVersion)) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 8c01ae36346..f3458ab24c9 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -178,9 +178,7 @@ class VolumeTracingController @Inject()( initialData <- request.body.asRaw.map(_.asFile) ?~> Messages("zipFile.notFound") tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") magRestrictions = MagRestrictions(minMag, maxMag) - mags <- volumeTracingService - .initializeWithData(annotationId, tracingId, tracing, initialData, magRestrictions) - .toFox + mags <- volumeTracingService.initializeWithData(tracingId, tracing, initialData, magRestrictions).toFox _ <- volumeTracingService.updateMagList(tracingId, tracing, mags) } yield Ok(Json.toJson(tracingId)) } @@ -218,9 +216,7 @@ class VolumeTracingController @Inject()( annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(tracingId) initialData <- request.body.asRaw.map(_.asFile) ?~> Messages("zipFile.notFound") tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") - mags <- volumeTracingService - .initializeWithDataMultiple(annotationId, tracingId, tracing, initialData) - .toFox + mags <- volumeTracingService.initializeWithDataMultiple(tracingId, tracing, initialData).toFox _ <- volumeTracingService.updateMagList(tracingId, tracing, mags) } yield Ok(Json.toJson(tracingId)) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala index c0f06df09cc..c458ec28d17 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala @@ -34,8 +34,6 @@ trait TracingService[T <: GeneratedMessage] def tracingMigrationService: TracingMigrationService[T] - def dummyTracing: T - implicit def tracingCompanion: GeneratedMessageCompanion[T] // this should be longer than maxCacheTime in webknossos/AnnotationStore diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 39523d44a08..be29e9fa621 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -313,7 +313,7 @@ class VolumeTracingService @Inject()( } yield () } - def initializeWithDataMultiple(annotationId: String, tracingId: String, tracing: VolumeTracing, initialData: File)( + def initializeWithDataMultiple(tracingId: String, tracing: VolumeTracing, initialData: File)( implicit mp: MessagesProvider, tc: TokenContext): Fox[Set[Vec3Int]] = if (tracing.version != 0L) @@ -382,8 +382,7 @@ class VolumeTracingService @Inject()( } yield mags } - def initializeWithData(annotationId: String, - tracingId: String, + def initializeWithData(tracingId: String, tracing: VolumeTracing, initialData: File, magRestrictions: MagRestrictions)(implicit tc: TokenContext): Fox[Set[Vec3Int]] = @@ -496,7 +495,7 @@ class VolumeTracingService @Inject()( editPosition: Option[Vec3Int], editRotation: Option[Vec3Double], newVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[VolumeTracing] = { - val tracingWithBB = addBoundingBoxFromTaskIfRequired(sourceTracing, isFromTask, boundingBox) + val tracingWithBB = addBoundingBoxFromTaskIfRequired(sourceTracing, isFromTask, datasetBoundingBox) val tracingWithMagRestrictions = restrictMagList(tracingWithBB, magRestrictions) for { fallbackLayer <- getFallbackLayer(sourceTracingId, sourceTracing) @@ -521,7 +520,7 @@ class VolumeTracingService @Inject()( isFromTask: Boolean, datasetBoundingBoxOpt: Option[BoundingBox]): VolumeTracing = datasetBoundingBoxOpt match { - case Some(datasetBoundingBox) if isFromTask => { + case Some(datasetBoundingBox) if isFromTask => val newId = if (tracing.userBoundingBoxes.isEmpty) 1 else tracing.userBoundingBoxes.map(_.id).max + 1 tracing .addUserBoundingBoxes( @@ -531,19 +530,19 @@ class VolumeTracingService @Inject()( Some(getRandomColor), tracing.boundingBox)) .withBoundingBox(datasetBoundingBox) - } case _ => tracing } def duplicateVolumeData(sourceTracingId: String, sourceTracing: VolumeTracing, newTracingId: String, - newTracing: VolumeTracing)(implicit tc: TokenContext): Fox[Unit] = + newTracing: VolumeTracing)(implicit tc: TokenContext): Fox[Unit] = { + var bucketCount = 0 for { isTemporaryTracing <- isTemporaryTracing(sourceTracingId) sourceDataLayer = volumeTracingLayer(sourceTracingId, sourceTracing, isTemporaryTracing) buckets: Iterator[(BucketPosition, Array[Byte])] = sourceDataLayer.bucketProvider.bucketStream( - Some(newTracing.version)) + Some(sourceTracing.version)) destinationDataLayer = volumeTracingLayer(newTracingId, newTracing) fallbackLayer <- getFallbackLayer(sourceTracingId, sourceTracing) segmentIndexBuffer = new VolumeSegmentIndexBuffer( @@ -561,6 +560,7 @@ class VolumeTracingService @Inject()( if (newTracing.mags.contains(vec3IntToProto(bucketPosition.mag))) { for { _ <- saveBucket(destinationDataLayer, bucketPosition, bucketData, newTracing.version) + _ = bucketCount += 1 _ <- Fox.runIfOptionTrue(newTracing.hasSegmentIndex)( updateSegmentIndex( segmentIndexBuffer, @@ -574,8 +574,11 @@ class VolumeTracingService @Inject()( } yield () } else Fox.successful(()) } + _ = logger.info( + s"Duplicated $bucketCount volume buckets from $sourceTracingId v${sourceTracing.version} to $newTracingId v${newTracing.version}.") _ <- segmentIndexBuffer.flush() } yield () + } private def volumeTracingLayer( tracingId: String, @@ -605,6 +608,7 @@ class VolumeTracingService @Inject()( toCache) } yield id + // TODO use or remove def downsample(annotationId: String, tracingId: String, oldTracingId: String, newTracing: VolumeTracing)( implicit tc: TokenContext): Fox[Unit] = for { @@ -784,7 +788,9 @@ class VolumeTracingService @Inject()( elementClass) mergedAdditionalAxes <- Fox.box2Fox(AdditionalAxis.mergeAndAssertSameAdditionalAxes(tracings.map(t => AdditionalAxis.fromProtosAsOpt(t.additionalAxes)))) - fallbackLayer <- getFallbackLayer(tracingSelectors.head.tracingId, tracings.head) // TODO can we get rid of the head? + firstTracingSelector <- tracingSelectors.headOption ?~> "merge.noTracings" + firstTracing <- tracings.headOption ?~> "merge.noTracings" + fallbackLayer <- getFallbackLayer(firstTracingSelector.tracingId, firstTracing) segmentIndexBuffer = new VolumeSegmentIndexBuffer(newId, volumeSegmentIndexClient, newVersion, @@ -882,8 +888,6 @@ class VolumeTracingService @Inject()( } } - def dummyTracing: VolumeTracing = ??? - def mergeEditableMappings(newTracingId: String, tracingsWithIds: List[(VolumeTracing, String)])( implicit tc: TokenContext): Fox[Unit] = if (tracingsWithIds.forall(tracingWithId => tracingWithId._1.getHasEditableMapping)) { @@ -892,7 +896,7 @@ class VolumeTracingService @Inject()( remoteFallbackLayerFromVolumeTracing(tracingWithId._1, tracingWithId._2)) remoteFallbackLayer <- remoteFallbackLayers.headOption.toFox _ <- bool2Fox(remoteFallbackLayers.forall(_ == remoteFallbackLayer)) ?~> "Cannot merge editable mappings based on different dataset layers" - // TODO_ <- editableMappingService.merge(newTracingId, tracingsWithIds.map(_._2), remoteFallbackLayer) + // TODO: _ <- editableMappingService.merge(newTracingId, tracingsWithIds.map(_._2), remoteFallbackLayer) } yield () } else if (tracingsWithIds.forall(tracingWithId => !tracingWithId._1.getHasEditableMapping)) { Fox.empty From 11cbcd294eb3ace4c33b898c5395f2394db5fe89 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 30 Oct 2024 09:55:38 +0100 Subject: [PATCH 136/150] rename annotation.proto layers to annotationLayers, simplify duplicate api --- app/controllers/AnnotationController.scala | 8 +-- .../WKRemoteTracingStoreController.scala | 2 +- app/models/annotation/AnnotationService.scala | 22 +++---- .../WKRemoteTracingStoreClient.scala | 15 +---- .../oxalis/model_initialization.ts | 2 +- frontend/javascripts/types/api_flow_types.ts | 2 +- webknossos-datastore/proto/Annotation.proto | 2 +- .../annotation/AnnotationWithTracings.scala | 10 +-- .../annotation/TSAnnotationService.scala | 62 ++++++------------- .../controllers/TSAnnotationController.scala | 18 +----- ...alableminds.webknossos.tracingstore.routes | 2 +- 11 files changed, 46 insertions(+), 99 deletions(-) diff --git a/app/controllers/AnnotationController.scala b/app/controllers/AnnotationController.scala index 689586935b4..62b64efea09 100755 --- a/app/controllers/AnnotationController.scala +++ b/app/controllers/AnnotationController.scala @@ -439,13 +439,9 @@ class AnnotationController @Inject()( newAnnotationId, version = None, isFromTask = annotation._task.isDefined, - editPosition = None, - editRotation = None, - boundingBox = None, - datasetBoundingBox = dataSource.map(_.boundingBox), - magRestrictions = MagRestrictions.empty + datasetBoundingBox = dataSource.map(_.boundingBox) ) - newAnnotationLayers = newAnnotationProto.layers.map(AnnotationLayer.fromProto) + newAnnotationLayers = newAnnotationProto.annotationLayers.map(AnnotationLayer.fromProto) clonedAnnotation <- annotationService.createFrom(user, dataset, newAnnotationLayers, diff --git a/app/controllers/WKRemoteTracingStoreController.scala b/app/controllers/WKRemoteTracingStoreController.scala index 1e05a4df3f5..b659700a52f 100644 --- a/app/controllers/WKRemoteTracingStoreController.scala +++ b/app/controllers/WKRemoteTracingStoreController.scala @@ -67,7 +67,7 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore for { annotationIdValidated <- ObjectId.fromString(annotationId) existingLayers <- annotationLayerDAO.findAnnotationLayersFor(annotationIdValidated) - newLayersProto = request.body.layers + newLayersProto = request.body.annotationLayers existingLayerIds = existingLayers.map(_.tracingId).toSet newLayerIds = newLayersProto.map(_.tracingId).toSet layerIdsToDelete = existingLayerIds.diff(newLayerIds) diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index d821d478068..759b8efe180 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -281,7 +281,7 @@ class AnnotationService @Inject()( annotationProto = AnnotationProto(name = Some(AnnotationDefaults.defaultName), description = Some(AnnotationDefaults.defaultDescription), version = 0L, - layers = layersProto) + annotationLayers = layersProto) _ <- tracingStoreClient.saveAnnotationProto(annotationId, annotationProto) } yield newAnnotationLayers @@ -370,17 +370,13 @@ class AnnotationService @Inject()( annotationBaseId, initializingAnnotationId, version = None, - isFromTask = false, - editPosition = None, - editRotation = None, - boundingBox = None, - datasetBoundingBox = None, - magRestrictions = MagRestrictions.empty + isFromTask = false, // isFromTask is when duplicate is called on a task annotation, not when a task is assigned + datasetBoundingBox = None ) newAnnotation = annotationBase.copy( _id = initializingAnnotationId, _user = user._id, - annotationLayers = duplicatedAnnotationProto.layers.map(AnnotationLayer.fromProto).toList, + annotationLayers = duplicatedAnnotationProto.annotationLayers.map(AnnotationLayer.fromProto).toList, state = Active, typ = AnnotationType.Task, created = Instant.now, @@ -485,10 +481,12 @@ class AnnotationService @Inject()( annotationLayers, description.getOrElse(""), typ = AnnotationType.TracingBase) - annotationBaseProto = AnnotationProto(name = Some(AnnotationDefaults.defaultName), - description = Some(AnnotationDefaults.defaultDescription), - version = 0L, - layers = annotationLayers.map(_.toProto)) + annotationBaseProto = AnnotationProto( + name = Some(AnnotationDefaults.defaultName), + description = Some(AnnotationDefaults.defaultDescription), + version = 0L, + annotationLayers = annotationLayers.map(_.toProto) + ) _ <- tracingStoreClient.saveAnnotationProto(annotationBase._id, annotationBaseProto) _ = logger.info(s"inserting base annotation ${annotationBase._id} for task ${task._id}") _ <- annotationDAO.insertOne(annotationBase) diff --git a/app/models/annotation/WKRemoteTracingStoreClient.scala b/app/models/annotation/WKRemoteTracingStoreClient.scala index 6f072bbfc8d..a5bb17dd70e 100644 --- a/app/models/annotation/WKRemoteTracingStoreClient.scala +++ b/app/models/annotation/WKRemoteTracingStoreClient.scala @@ -84,7 +84,7 @@ class WKRemoteTracingStoreClient( def saveAnnotationProto(annotationId: ObjectId, annotationProto: AnnotationProto): Fox[Unit] = { logger.debug( - f"Called to save AnnotationProto $annotationId with layers ${annotationProto.layers.map(_.tracingId).mkString(",")}." + baseInfo) + f"Called to save AnnotationProto $annotationId with layers ${annotationProto.annotationLayers.map(_.tracingId).mkString(",")}." + baseInfo) rpc(s"${tracingStore.url}/tracings/annotation/save") .addQueryString("token" -> RpcTokenHolder.webknossosToken) .addQueryString("annotationId" -> annotationId.toString) @@ -92,29 +92,18 @@ class WKRemoteTracingStoreClient( } // Used in duplicate route. History and version are kept - // TODO: can we remove some params here, if they are used only in task case? def duplicateAnnotation(annotationId: ObjectId, newAnnotationId: ObjectId, version: Option[Long], isFromTask: Boolean, - editPosition: Option[Vec3Int], - editRotation: Option[Vec3Double], - boundingBox: Option[BoundingBox], - datasetBoundingBox: Option[BoundingBox], - magRestrictions: MagRestrictions, - ): Fox[AnnotationProto] = { + datasetBoundingBox: Option[BoundingBox]): Fox[AnnotationProto] = { logger.debug(s"Called to duplicate annotation $annotationId." + baseInfo) rpc(s"${tracingStore.url}/tracings/annotation/$annotationId/duplicate").withLongTimeout .addQueryString("token" -> RpcTokenHolder.webknossosToken) .addQueryString("newAnnotationId" -> newAnnotationId.toString) .addQueryStringOptional("version", version.map(_.toString)) - .addQueryStringOptional("editPosition", editPosition.map(_.toUriLiteral)) - .addQueryStringOptional("editRotation", editRotation.map(_.toUriLiteral)) - .addQueryStringOptional("boundingBox", boundingBox.map(_.toLiteral)) .addQueryStringOptional("datasetBoundingBox", datasetBoundingBox.map(_.toLiteral)) .addQueryString("isFromTask" -> isFromTask.toString) - .addQueryStringOptional("minMag", magRestrictions.minStr) - .addQueryStringOptional("maxMag", magRestrictions.maxStr) .postWithProtoResponse[AnnotationProto]()(AnnotationProto) } diff --git a/frontend/javascripts/oxalis/model_initialization.ts b/frontend/javascripts/oxalis/model_initialization.ts index e753c152c7e..baa43bccb2d 100644 --- a/frontend/javascripts/oxalis/model_initialization.ts +++ b/frontend/javascripts/oxalis/model_initialization.ts @@ -143,7 +143,7 @@ export async function initialize( name: annotationFromTracingStore.name, description: annotationFromTracingStore.description, }; - annotationFromTracingStore.layers.forEach((layer) => { + annotationFromTracingStore.annotationLayers.forEach((layer) => { if ( maybeOutdatedAnnotation.annotationLayers.find((l) => l.tracingId === layer.tracingId) == null diff --git a/frontend/javascripts/types/api_flow_types.ts b/frontend/javascripts/types/api_flow_types.ts index ee1a3d92706..b2c564e91b1 100644 --- a/frontend/javascripts/types/api_flow_types.ts +++ b/frontend/javascripts/types/api_flow_types.ts @@ -585,7 +585,7 @@ export type APITracingStoreAnnotation = { name: string; description: string; version: number; - layers: APITracingStoreAnnotationLayer[]; + annotationLayers: APITracingStoreAnnotationLayer[]; }; export type APITimeTrackingPerUser = { diff --git a/webknossos-datastore/proto/Annotation.proto b/webknossos-datastore/proto/Annotation.proto index e8938373307..5eaab3982f7 100644 --- a/webknossos-datastore/proto/Annotation.proto +++ b/webknossos-datastore/proto/Annotation.proto @@ -11,7 +11,7 @@ message AnnotationProto { optional string name = 1; optional string description = 2; required int64 version = 3; - repeated AnnotationLayerProto layers = 4; + repeated AnnotationLayerProto annotationLayers = 4; } message AnnotationLayerProto { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala index 482ebb5b12c..15563fab4a0 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationWithTracings.scala @@ -83,7 +83,7 @@ case class AnnotationWithTracings( tracing: Either[SkeletonTracing, VolumeTracing]): AnnotationWithTracings = this.copy( annotation = annotation.copy( - layers = annotation.layers :+ AnnotationLayerProto( + annotationLayers = annotation.annotationLayers :+ AnnotationLayerProto( tracingId, a.layerParameters.name.getOrElse(AnnotationLayer.defaultNameForType(a.layerParameters.typ)), `type` = AnnotationLayerType.toProto(a.layerParameters.typ) @@ -92,11 +92,13 @@ case class AnnotationWithTracings( ) def deleteTracing(a: DeleteLayerAnnotationAction): AnnotationWithTracings = - this.copy(annotation = annotation.copy(layers = annotation.layers.filter(_.tracingId != a.tracingId)), - tracingsById = tracingsById.removed(a.tracingId)) + this.copy( + annotation = annotation.copy(annotationLayers = annotation.annotationLayers.filter(_.tracingId != a.tracingId)), + tracingsById = tracingsById.removed(a.tracingId) + ) def updateLayerMetadata(a: UpdateLayerMetadataAnnotationAction): AnnotationWithTracings = - this.copy(annotation = annotation.copy(layers = annotation.layers.map(l => + this.copy(annotation = annotation.copy(annotationLayers = annotation.annotationLayers.map(l => if (l.tracingId == a.tracingId) l.copy(name = a.layerName) else l))) def updateMetadata(a: UpdateMetadataAnnotationAction): AnnotationWithTracings = diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index a449d6d80d0..675919921bc 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -147,10 +147,10 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss for { tracingId <- action.tracingId.toFox ?~> "add layer action has no tracingId" _ <- bool2Fox( - !annotationWithTracings.annotation.layers + !annotationWithTracings.annotation.annotationLayers .exists(_.name == action.layerParameters.getNameWithDefault)) ?~> "addLayer.nameInUse" _ <- bool2Fox( - !annotationWithTracings.annotation.layers.exists( + !annotationWithTracings.annotation.annotationLayers.exists( _.`type` == AnnotationLayerTypeProto.Skeleton && action.layerParameters.typ == AnnotationLayerType.Skeleton)) ?~> "addLayer.onlyOneSkeletonAllowed" tracing <- remoteWebknossosClient.createTracingFor(annotationId, action.layerParameters, @@ -366,7 +366,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss requestAll: Boolean)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = { val skeletonTracingIds = if (requestAll) - annotation.layers.filter(_.`type` == AnnotationLayerTypeProto.Skeleton).map(_.tracingId) + annotation.annotationLayers.filter(_.`type` == AnnotationLayerTypeProto.Skeleton).map(_.tracingId) else { (updates.flatMap { case u: SkeletonUpdateAction => Some(u.actionTracingId) @@ -375,7 +375,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } val volumeTracingIds = if (requestAll) - annotation.layers.filter(_.`type` == AnnotationLayerTypeProto.Volume).map(_.tracingId) + annotation.annotationLayers.filter(_.`type` == AnnotationLayerTypeProto.Volume).map(_.tracingId) else { (updates.flatMap { case u: VolumeUpdateAction => Some(u.actionTracingId) @@ -622,26 +622,13 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss newAnnotationId: String, version: Option[Long], isFromTask: Boolean, - editPosition: Option[Vec3Int], - editRotation: Option[Vec3Double], - boundingBox: Option[BoundingBox], - datasetBoundingBox: Option[BoundingBox], - magRestrictions: MagRestrictions)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationProto] = + datasetBoundingBox: Option[BoundingBox])(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationProto] = for { currentAnnotation <- get(annotationId, version) - newLayers <- Fox.serialCombined(currentAnnotation.layers)( - layer => - duplicateLayer(annotationId, - layer, - currentAnnotation.version, - isFromTask, - editPosition, - editRotation, - boundingBox, - datasetBoundingBox, - magRestrictions)) + newLayers <- Fox.serialCombined(currentAnnotation.annotationLayers)(layer => + duplicateLayer(annotationId, layer, currentAnnotation.version, isFromTask, datasetBoundingBox)) _ <- duplicateUpdates(annotationId, newAnnotationId) - duplicatedAnnotation = currentAnnotation.copy(layers = newLayers) + duplicatedAnnotation = currentAnnotation.copy(annotationLayers = newLayers) _ <- tracingDataStore.annotations.put(newAnnotationId, currentAnnotation.version, duplicatedAnnotation) } yield duplicatedAnnotation @@ -657,16 +644,12 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } } yield () - private def duplicateLayer( - annotationId: String, - layer: AnnotationLayerProto, - version: Long, - isFromTask: Boolean, - editPosition: Option[Vec3Int], - editRotation: Option[Vec3Double], - boundingBox: Option[BoundingBox], - datasetBoundingBox: Option[BoundingBox], - magRestrictions: MagRestrictions)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationLayerProto] = + private def duplicateLayer(annotationId: String, + layer: AnnotationLayerProto, + version: Long, + isFromTask: Boolean, + datasetBoundingBox: Option[BoundingBox])(implicit ec: ExecutionContext, + tc: TokenContext): Fox[AnnotationLayerProto] = for { newTracingId <- layer.`type` match { case AnnotationLayerTypeProto.Volume => @@ -675,20 +658,13 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss version, version, isFromTask, - boundingBox, + None, datasetBoundingBox, - magRestrictions, - editPosition, - editRotation) + MagRestrictions.empty, + None, + None) case AnnotationLayerTypeProto.Skeleton => - duplicateSkeletonTracing(annotationId, - layer.tracingId, - version, - version, - isFromTask, - editPosition, - editRotation, - boundingBox) + duplicateSkeletonTracing(annotationId, layer.tracingId, version, version, isFromTask, None, None, None) case AnnotationLayerTypeProto.Unrecognized(num) => Fox.failure(f"unrecognized annotation layer type: $num") } } yield layer.copy(tracingId = newTracingId) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index c27795c0cb4..6914d4aadf7 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -1,7 +1,7 @@ package com.scalableminds.webknossos.tracingstore.controllers import com.google.inject.Inject -import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int} +import com.scalableminds.util.geometry.BoundingBox import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.Annotation.AnnotationProto import com.scalableminds.webknossos.datastore.controllers.Controller @@ -14,7 +14,6 @@ import com.scalableminds.webknossos.tracingstore.annotation.{ UpdateActionGroup } import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService -import com.scalableminds.webknossos.tracingstore.tracings.volume.MagRestrictions import play.api.libs.json.Json import play.api.mvc.{Action, AnyContent, PlayBodyParsers} @@ -105,31 +104,18 @@ class TSAnnotationController @Inject()( newAnnotationId: String, version: Option[Long], isFromTask: Boolean, - minMag: Option[Int], - maxMag: Option[Int], - editPosition: Option[String], - editRotation: Option[String], - boundingBox: Option[String], datasetBoundingBox: Option[String]): Action[AnyContent] = Action.async { implicit request => log() { logTime(slackNotificationService.noticeSlowRequest) { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readAnnotation(annotationId)) { for { - editPositionParsed <- Fox.runOptional(editPosition)(Vec3Int.fromUriLiteral) - editRotationParsed <- Fox.runOptional(editRotation)(Vec3Double.fromUriLiteral) - boundingBoxParsed <- Fox.runOptional(boundingBox)(BoundingBox.fromLiteral) datasetBoundingBoxParsed <- Fox.runOptional(datasetBoundingBox)(BoundingBox.fromLiteral) - magRestrictions = MagRestrictions(minMag, maxMag) annotationProto <- annotationService.duplicate(annotationId, newAnnotationId, version, isFromTask, - editPositionParsed, - editRotationParsed, - boundingBoxParsed, - datasetBoundingBoxParsed, - magRestrictions) + datasetBoundingBoxParsed) } yield Ok(annotationProto.toByteArray).as(protobufMimeType) } } diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 06b9967a84c..612798c2a1e 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -11,7 +11,7 @@ POST /annotation/:annotationId/update GET /annotation/:annotationId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionLog(annotationId: String, newestVersion: Option[Long], oldestVersion: Option[Long]) GET /annotation/:annotationId/updateActionStatistics @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionStatistics(annotationId: String) GET /annotation/:annotationId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.newestVersion(annotationId: String) -POST /annotation/:annotationId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.duplicate(annotationId: String, newAnnotationId: String, version: Option[Long], isFromTask: Boolean, minMag: Option[Int], maxMag: Option[Int], editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String], datasetBoundingBox: Option[String]) +POST /annotation/:annotationId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.duplicate(annotationId: String, newAnnotationId: String, version: Option[Long], isFromTask: Boolean, datasetBoundingBox: Option[String]) # Volume tracings POST /volume/save @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.save() From 65f2ebd979701e0a428dd9aa300c60923661cef3 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 30 Oct 2024 10:26:00 +0100 Subject: [PATCH 137/150] wip merge --- .../com/scalableminds/util/tools/Fox.scala | 5 +- .../annotation/TSAnnotationService.scala | 10 +++- .../SkeletonTracingController.scala | 6 +-- .../controllers/TSAnnotationController.scala | 50 ++++++++++++++++++- .../tracings/TracingService.scala | 2 - .../skeleton/SkeletonTracingService.scala | 4 +- ...alableminds.webknossos.tracingstore.routes | 1 + 7 files changed, 63 insertions(+), 15 deletions(-) diff --git a/util/src/main/scala/com/scalableminds/util/tools/Fox.scala b/util/src/main/scala/com/scalableminds/util/tools/Fox.scala index 2195b751eef..953ed2b7f5b 100644 --- a/util/src/main/scala/com/scalableminds/util/tools/Fox.scala +++ b/util/src/main/scala/com/scalableminds/util/tools/Fox.scala @@ -101,13 +101,14 @@ object Fox extends FoxImplicits { def sequence[T](l: List[Fox[T]])(implicit ec: ExecutionContext): Future[List[Box[T]]] = Future.sequence(l.map(_.futureBox)) - def combined[T](l: List[Fox[T]])(implicit ec: ExecutionContext): Fox[List[T]] = + def combined[T](l: Seq[Fox[T]])(implicit ec: ExecutionContext): Fox[List[T]] = Fox(Future.sequence(l.map(_.futureBox)).map { results => results.find(_.isEmpty) match { case Some(Empty) => Empty case Some(failure: Failure) => failure case _ => - Full(results.map(_.openOrThrowException("An exception should never be thrown, all boxes must be full"))) + Full( + results.map(_.openOrThrowException("An exception should never be thrown, all boxes must be full")).toList) } }) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 675919921bc..ae49dafaec4 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -203,6 +203,12 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss withTracings <- getWithTracings(annotationId, version, List.empty, List.empty, requestAll = false) } yield withTracings.annotation + def getMultiple(annotationIds: Seq[String])(implicit ec: ExecutionContext, + tc: TokenContext): Fox[Seq[AnnotationProto]] = + Fox.serialCombined(annotationIds) { annotationId => + get(annotationId, None) + } + private def getWithTracings( annotationId: String, version: Option[Long], @@ -600,7 +606,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } } - def findMultipleSkeletons(selectors: List[Option[TracingSelector]], + def findMultipleSkeletons(selectors: Seq[Option[TracingSelector]], useCache: Boolean = true, applyUpdates: Boolean = false)(implicit tc: TokenContext, ec: ExecutionContext): Fox[List[Option[SkeletonTracing]]] = @@ -608,7 +614,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss selectors.map { case Some(selector) => for { - annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(selector.tracingId) + annotationId <- remoteWebknossosClient.getAnnotationIdForTracing(selector.tracingId) // TODO perf skip that if we already have it? tracing <- findSkeleton(annotationId, selector.tracingId, selector.version, useCache, applyUpdates) .map(Some(_)) } yield tracing diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala index 40fca232591..5636b68ae02 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala @@ -120,8 +120,7 @@ class SkeletonTracingController @Inject()(skeletonTracingService: SkeletonTracin case Empty => Fox.successful(None) case f: Failure => f.toFox } - mergedTracing <- Fox.box2Fox( - skeletonTracingService.merge(tracingsWithIds.map(_._1), mergedVolumeStats, newEditableMappingIdOpt)) + mergedTracing <- Fox.box2Fox(skeletonTracingService.merge(tracingsWithIds.map(_._1))) _ <- skeletonTracingService.save(mergedTracing, Some(newTracingId), version = 0, toCache = !persist) } yield Ok(Json.toJson(newTracingId)) } @@ -134,8 +133,7 @@ class SkeletonTracingController @Inject()(skeletonTracingService: SkeletonTracin accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { val tracings: List[Option[SkeletonTracing]] = request.body for { - mergedTracing <- Fox.box2Fox( - skeletonTracingService.merge(tracings.flatten, MergedVolumeStats.empty(), Empty)) + mergedTracing <- Fox.box2Fox(skeletonTracingService.merge(tracings.flatten)) processedTracing = skeletonTracingService.remapTooLargeTreeIds(mergedTracing) newId <- skeletonTracingService.save(processedTracing, None, processedTracing.version, toCache = !persist) } yield Ok(Json.toJson(newId)) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index 6914d4aadf7..ae3ab5dca3f 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -3,10 +3,15 @@ package com.scalableminds.webknossos.tracingstore.controllers import com.google.inject.Inject import com.scalableminds.util.geometry.BoundingBox import com.scalableminds.util.tools.Fox -import com.scalableminds.webknossos.datastore.Annotation.AnnotationProto +import com.scalableminds.webknossos.datastore.Annotation.{AnnotationLayerTypeProto, AnnotationProto} import com.scalableminds.webknossos.datastore.controllers.Controller import com.scalableminds.webknossos.datastore.services.UserAccessRequest -import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore} +import com.scalableminds.webknossos.tracingstore.tracings.{ + KeyValueStoreImplicits, + TracingDataStore, + TracingId, + TracingSelector +} import com.scalableminds.webknossos.tracingstore.TracingStoreAccessTokenService import com.scalableminds.webknossos.tracingstore.annotation.{ AnnotationTransactionService, @@ -14,6 +19,8 @@ import com.scalableminds.webknossos.tracingstore.annotation.{ UpdateActionGroup } import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService +import com.scalableminds.webknossos.tracingstore.tracings.skeleton.SkeletonTracingService +import play.api.i18n.Messages import play.api.libs.json.Json import play.api.mvc.{Action, AnyContent, PlayBodyParsers} @@ -24,6 +31,7 @@ class TSAnnotationController @Inject()( slackNotificationService: TSSlackNotificationService, annotationService: TSAnnotationService, annotationTransactionService: AnnotationTransactionService, + skeletonTracingService: SkeletonTracingService, tracingDataStore: TracingDataStore)(implicit ec: ExecutionContext, bodyParsers: PlayBodyParsers) extends Controller with KeyValueStoreImplicits { @@ -121,4 +129,42 @@ class TSAnnotationController @Inject()( } } } + + def mergedFromIds(persist: Boolean, newAnnotationId: String): Action[List[String]] = + Action.async(validateJson[List[String]]) { implicit request => + log() { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { + for { + annotations: Seq[AnnotationProto] <- annotationService.getMultiple(request.body) ?~> Messages( + "annotation.notFound") + annotationsWithIds = annotations.zip(annotations) + skeletonIds = annotations.flatMap( + _.annotationLayers.filter(_.`type` == AnnotationLayerTypeProto.Skeleton).map(_.tracingId)) + volumeIds = annotations.flatMap( + _.annotationLayers.filter(_.`type` == AnnotationLayerTypeProto.Skeleton).map(_.tracingId)) + /*mergedVolumeStats <- volumeTracingService.mergeVolumeData(request.body.flatten, + tracingsWithIds.map(_._1), + newTracingId, + newVersion = 0L, + toCache = !persist) + mergeEditableMappingsResultBox <- skeletonTracingService + .mergeEditableMappings(newTracingId, tracingsWithIds) + .futureBox + newEditableMappingIdOpt <- mergeEditableMappingsResultBox match { + case Full(()) => Fox.successful(Some(newTracingId)) + case Empty => Fox.successful(None) + case f: Failure => f.toFox + }*/ + skeletons <- annotationService.findMultipleSkeletons(skeletonIds.map { s => + Some(TracingSelector(s)) + }, applyUpdates = true) + // TODO handle zero-skeletons / zero-volumes case + newSkeletonId = TracingId.generate + newVolumeId = TracingId.generate + mergedSkeleton <- skeletonTracingService.merge(skeletons.flatten).toFox + _ <- skeletonTracingService.save(mergedSkeleton, Some(newSkeletonId), version = 0, toCache = !persist) + } yield Ok + } + } + } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala index c458ec28d17..9c5dcae0db1 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala @@ -69,8 +69,6 @@ trait TracingService[T <: GeneratedMessage] } } - def merge(tracings: Seq[T], mergedVolumeStats: MergedVolumeStats, newEditableMappingIdOpt: Option[String]): Box[T] - def remapTooLargeTreeIds(tracing: T): T = tracing def mergeVolumeData(tracingSelectors: Seq[TracingSelector], diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala index 700c13c41c5..9c49183a7c8 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala @@ -61,9 +61,7 @@ class SkeletonTracingService @Inject()( if (fromTask) newTracing.clearBoundingBox else newTracing } - def merge(tracings: Seq[SkeletonTracing], - mergedVolumeStats: MergedVolumeStats, - newEditableMappingIdOpt: Option[String]): Box[SkeletonTracing] = + def merge(tracings: Seq[SkeletonTracing]): Box[SkeletonTracing] = for { tracing <- tracings.map(Full(_)).reduceLeft(mergeTwo) } yield diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 612798c2a1e..d1bb507338b 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -12,6 +12,7 @@ GET /annotation/:annotationId/updateActionLog GET /annotation/:annotationId/updateActionStatistics @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionStatistics(annotationId: String) GET /annotation/:annotationId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.newestVersion(annotationId: String) POST /annotation/:annotationId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.duplicate(annotationId: String, newAnnotationId: String, version: Option[Long], isFromTask: Boolean, datasetBoundingBox: Option[String]) +POST /annotation/mergedFromIds @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.mergedFromIds(persist: Boolean, newAnnotationId: String) # Volume tracings POST /volume/save @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.save() From e55768d88ab5dd1538c5a861fb27705deea50bbd Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 30 Oct 2024 11:14:05 +0100 Subject: [PATCH 138/150] merge skeletons --- app/models/annotation/AnnotationMerger.scala | 64 +++---------------- .../WKRemoteTracingStoreClient.scala | 11 ++++ .../annotation/TSAnnotationService.scala | 1 + .../controllers/TSAnnotationController.scala | 51 +++++++++++---- 4 files changed, 61 insertions(+), 66 deletions(-) diff --git a/app/models/annotation/AnnotationMerger.scala b/app/models/annotation/AnnotationMerger.scala index 2c251f01462..ec538208b15 100644 --- a/app/models/annotation/AnnotationMerger.scala +++ b/app/models/annotation/AnnotationMerger.scala @@ -2,11 +2,7 @@ package models.annotation import com.scalableminds.util.accesscontext.DBAccessContext import com.scalableminds.util.tools.{Fox, FoxImplicits} -import com.scalableminds.webknossos.datastore.models.annotation.{ - AnnotationLayer, - AnnotationLayerStatistics, - AnnotationLayerType -} +import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayer import com.typesafe.scalalogging.LazyLogging import javax.inject.Inject @@ -51,7 +47,7 @@ class AnnotationMerger @Inject()(datasetDAO: DatasetDAO, tracingStoreService: Tr Fox.empty else { for { - mergedAnnotationLayers <- mergeTracingsOfAnnotations(annotations, datasetId, persistTracing) + mergedAnnotationLayers <- mergeAnnotationsInTracingstore(annotations, datasetId, newId, persistTracing) ?~> "Failed to merge annotations in tracingstore." } yield { Annotation( newId, @@ -65,56 +61,16 @@ class AnnotationMerger @Inject()(datasetDAO: DatasetDAO, tracingStoreService: Tr } } - private def mergeTracingsOfAnnotations(annotations: List[Annotation], datasetId: ObjectId, persistTracing: Boolean)( - implicit ctx: DBAccessContext): Fox[List[AnnotationLayer]] = + private def mergeAnnotationsInTracingstore( + annotations: List[Annotation], + datasetId: ObjectId, + newAnnotationId: ObjectId, + persist: Boolean)(implicit ctx: DBAccessContext): Fox[List[AnnotationLayer]] = for { dataset <- datasetDAO.findOne(datasetId) tracingStoreClient: WKRemoteTracingStoreClient <- tracingStoreService.clientFor(dataset) - skeletonLayers = annotations.flatMap(_.annotationLayers.find(_.typ == AnnotationLayerType.Skeleton)) - volumeLayers = annotations.flatMap(_.annotationLayers.find(_.typ == AnnotationLayerType.Volume)) - mergedSkeletonTracingId <- mergeSkeletonTracings(tracingStoreClient, - skeletonLayers.map(_.tracingId), - persistTracing) - mergedVolumeTracingId <- mergeVolumeTracings(tracingStoreClient, volumeLayers.map(_.tracingId), persistTracing) - mergedSkeletonName = allEqual(skeletonLayers.map(_.name)) - mergedVolumeName = allEqual(volumeLayers.map(_.name)) - mergedSkeletonLayer = mergedSkeletonTracingId.map( - id => - AnnotationLayer(id, - AnnotationLayerType.Skeleton, - mergedSkeletonName.getOrElse(AnnotationLayer.defaultSkeletonLayerName), - AnnotationLayerStatistics.unknown)) - mergedVolumeLayer = mergedVolumeTracingId.map( - id => - AnnotationLayer(id, - AnnotationLayerType.Volume, - mergedVolumeName.getOrElse(AnnotationLayer.defaultVolumeLayerName), - AnnotationLayerStatistics.unknown)) - } yield List(mergedSkeletonLayer, mergedVolumeLayer).flatten - - private def allEqual(str: List[String]): Option[String] = - // returns the str if all names are equal, None otherwise - str.headOption.map(name => str.forall(_ == name)).flatMap { _ => - str.headOption - } - - private def mergeSkeletonTracings(tracingStoreClient: WKRemoteTracingStoreClient, - skeletonTracingIds: List[String], - persistTracing: Boolean) = - if (skeletonTracingIds.isEmpty) - Fox.successful(None) - else - tracingStoreClient - .mergeSkeletonTracingsByIds(skeletonTracingIds, persistTracing) - .map(Some(_)) ?~> "Failed to merge skeleton tracings." + mergedAnnotationProto <- tracingStoreClient.mergeAnnotationsByIds(annotations.map(_.id), newAnnotationId, persist) + layers = mergedAnnotationProto.annotationLayers.map(AnnotationLayer.fromProto) + } yield layers.toList - private def mergeVolumeTracings(tracingStoreClient: WKRemoteTracingStoreClient, - volumeTracingIds: List[String], - persistTracing: Boolean) = - if (volumeTracingIds.isEmpty) - Fox.successful(None) - else - tracingStoreClient - .mergeVolumeTracingsByIds(volumeTracingIds, persistTracing) - .map(Some(_)) ?~> "Failed to merge volume tracings." } diff --git a/app/models/annotation/WKRemoteTracingStoreClient.scala b/app/models/annotation/WKRemoteTracingStoreClient.scala index a5bb17dd70e..f7b42910bb6 100644 --- a/app/models/annotation/WKRemoteTracingStoreClient.scala +++ b/app/models/annotation/WKRemoteTracingStoreClient.scala @@ -136,6 +136,17 @@ class WKRemoteTracingStoreClient( .addQueryStringOptional("maxMag", magRestrictions.maxStr) .postWithJsonResponse[String]() + def mergeAnnotationsByIds(annotationIds: List[String], + newAnnotationId: ObjectId, + persist: Boolean): Fox[AnnotationProto] = { + logger.debug(s"Called to merge ${annotationIds.length} annotations by ids." + baseInfo) + rpc(s"${tracingStore.url}/tracings/annotation/mergedFromIds").withLongTimeout + .addQueryString("token" -> RpcTokenHolder.webknossosToken) + .addQueryString("persist" -> persist.toString) + .addQueryString("newAnnotationId" -> newAnnotationId.toString) + .postJsonWithProtoResponse[List[String], AnnotationProto](annotationIds)(AnnotationProto) + } + def mergeSkeletonTracingsByIds(tracingIds: List[String], persistTracing: Boolean): Fox[String] = { logger.debug("Called to merge SkeletonTracings by ids." + baseInfo) rpc(s"${tracingStore.url}/tracings/skeleton/mergedFromIds").withLongTimeout diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index ae49dafaec4..8f673d66862 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -606,6 +606,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } } + // TODO build variant without TracingSelector and Option? def findMultipleSkeletons(selectors: Seq[Option[TracingSelector]], useCache: Boolean = true, applyUpdates: Boolean = false)(implicit tc: TokenContext, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index ae3ab5dca3f..a4ce7ec82a8 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -3,8 +3,13 @@ package com.scalableminds.webknossos.tracingstore.controllers import com.google.inject.Inject import com.scalableminds.util.geometry.BoundingBox import com.scalableminds.util.tools.Fox -import com.scalableminds.webknossos.datastore.Annotation.{AnnotationLayerTypeProto, AnnotationProto} +import com.scalableminds.webknossos.datastore.Annotation.{ + AnnotationLayerProto, + AnnotationLayerTypeProto, + AnnotationProto +} import com.scalableminds.webknossos.datastore.controllers.Controller +import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayer import com.scalableminds.webknossos.datastore.services.UserAccessRequest import com.scalableminds.webknossos.tracingstore.tracings.{ KeyValueStoreImplicits, @@ -137,11 +142,12 @@ class TSAnnotationController @Inject()( for { annotations: Seq[AnnotationProto] <- annotationService.getMultiple(request.body) ?~> Messages( "annotation.notFound") - annotationsWithIds = annotations.zip(annotations) - skeletonIds = annotations.flatMap( - _.annotationLayers.filter(_.`type` == AnnotationLayerTypeProto.Skeleton).map(_.tracingId)) - volumeIds = annotations.flatMap( - _.annotationLayers.filter(_.`type` == AnnotationLayerTypeProto.Skeleton).map(_.tracingId)) + skeletonLayers = annotations.flatMap( + _.annotationLayers.filter(_.`type` == AnnotationLayerTypeProto.Skeleton)) + volumeLayers = annotations.flatMap(_.annotationLayers.filter(_.`type` == AnnotationLayerTypeProto.Volume)) + // TODO: Volume + // TODO: Merge updates? if so, iron out reverts? + // TODO: Merge editable mappings /*mergedVolumeStats <- volumeTracingService.mergeVolumeData(request.body.flatten, tracingsWithIds.map(_._1), newTracingId, @@ -155,16 +161,37 @@ class TSAnnotationController @Inject()( case Empty => Fox.successful(None) case f: Failure => f.toFox }*/ - skeletons <- annotationService.findMultipleSkeletons(skeletonIds.map { s => - Some(TracingSelector(s)) + skeletons <- annotationService.findMultipleSkeletons(skeletonLayers.map { l => + Some(TracingSelector(l.tracingId)) }, applyUpdates = true) - // TODO handle zero-skeletons / zero-volumes case newSkeletonId = TracingId.generate newVolumeId = TracingId.generate - mergedSkeleton <- skeletonTracingService.merge(skeletons.flatten).toFox - _ <- skeletonTracingService.save(mergedSkeleton, Some(newSkeletonId), version = 0, toCache = !persist) - } yield Ok + mergedSkeletonName = allEqual(skeletonLayers.map(_.name)) + .getOrElse(AnnotationLayer.defaultSkeletonLayerName) + mergedVolumeName = allEqual(volumeLayers.map(_.name)).getOrElse(AnnotationLayer.defaultVolumeLayerName) + mergedSkeletonOpt <- Fox.runIf(skeletons.flatten.nonEmpty)( + skeletonTracingService.merge(skeletons.flatten).toFox) + mergedSkeletonLayerOpt: Option[AnnotationLayerProto] = mergedSkeletonOpt.map( + _ => + AnnotationLayerProto(name = mergedSkeletonName, + tracingId = newSkeletonId, + `type` = AnnotationLayerTypeProto.Skeleton)) + mergedVolumeLayerOpt: Option[AnnotationLayerProto] = None // TODO + mergedLayers = Seq(mergedSkeletonLayerOpt, mergedVolumeLayerOpt).flatten + firstAnnotation <- annotations.headOption.toFox + mergedAnnotation = firstAnnotation.withAnnotationLayers(mergedLayers) + _ <- Fox.runOptional(mergedSkeletonOpt)( + skeletonTracingService.save(_, Some(newSkeletonId), version = 0L, toCache = !persist)) + _ <- tracingDataStore.annotations.put(newAnnotationId, 0L, mergedAnnotation) + } yield Ok(mergedAnnotation.toByteArray).as(protobufMimeType) } } } + + // TODO generalize, mix with assertAllOnSame* + private def allEqual(str: Seq[String]): Option[String] = + // returns the str if all names are equal, None otherwise + str.headOption.map(name => str.forall(_ == name)).flatMap { _ => + str.headOption + } } From 63c8e6f2acdaa34894a58d25b321aa441b3da9b4 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 30 Oct 2024 11:35:56 +0100 Subject: [PATCH 139/150] merge volume data --- .../WKRemoteTracingStoreClient.scala | 16 ----- .../annotation/TSAnnotationService.scala | 2 +- .../SkeletonTracingController.scala | 36 +---------- .../controllers/TSAnnotationController.scala | 62 ++++++++++++------- .../controllers/VolumeTracingController.scala | 37 +---------- .../tracings/TracingService.scala | 14 ----- .../skeleton/SkeletonTracingService.scala | 17 +---- .../volume/VolumeTracingService.scala | 38 +++++++----- ...alableminds.webknossos.tracingstore.routes | 2 - 9 files changed, 68 insertions(+), 156 deletions(-) diff --git a/app/models/annotation/WKRemoteTracingStoreClient.scala b/app/models/annotation/WKRemoteTracingStoreClient.scala index f7b42910bb6..b832300245f 100644 --- a/app/models/annotation/WKRemoteTracingStoreClient.scala +++ b/app/models/annotation/WKRemoteTracingStoreClient.scala @@ -147,22 +147,6 @@ class WKRemoteTracingStoreClient( .postJsonWithProtoResponse[List[String], AnnotationProto](annotationIds)(AnnotationProto) } - def mergeSkeletonTracingsByIds(tracingIds: List[String], persistTracing: Boolean): Fox[String] = { - logger.debug("Called to merge SkeletonTracings by ids." + baseInfo) - rpc(s"${tracingStore.url}/tracings/skeleton/mergedFromIds").withLongTimeout - .addQueryString("token" -> RpcTokenHolder.webknossosToken) - .addQueryString("persist" -> persistTracing.toString) - .postJsonWithJsonResponse[List[TracingSelector], String](tracingIds.map(TracingSelector(_))) - } - - def mergeVolumeTracingsByIds(tracingIds: List[String], persistTracing: Boolean): Fox[String] = { - logger.debug("Called to merge VolumeTracings by ids." + baseInfo) - rpc(s"${tracingStore.url}/tracings/volume/mergedFromIds").withLongTimeout - .addQueryString("token" -> RpcTokenHolder.webknossosToken) - .addQueryString("persist" -> persistTracing.toString) - .postJsonWithJsonResponse[List[TracingSelector], String](tracingIds.map(TracingSelector(_))) - } - def mergeSkeletonTracingsByContents(tracings: SkeletonTracings, persistTracing: Boolean): Fox[String] = { logger.debug("Called to merge SkeletonTracings by contents." + baseInfo) rpc(s"${tracingStore.url}/tracings/skeleton/mergedFromContents").withLongTimeout diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 8f673d66862..9bf7ff95522 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -590,7 +590,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } yield tracing } - def findMultipleVolumes(selectors: List[Option[TracingSelector]], + def findMultipleVolumes(selectors: Seq[Option[TracingSelector]], useCache: Boolean = true, applyUpdates: Boolean = false)(implicit tc: TokenContext, ec: ExecutionContext): Fox[List[Option[VolumeTracing]]] = diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala index 5636b68ae02..17893f1670b 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala @@ -6,11 +6,9 @@ import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.SkeletonTracing.{SkeletonTracing, SkeletonTracingOpt, SkeletonTracings} import com.scalableminds.webknossos.datastore.services.UserAccessRequest import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService -import com.scalableminds.webknossos.tracingstore.tracings.{TracingId, TracingSelector} +import com.scalableminds.webknossos.tracingstore.tracings.TracingSelector import com.scalableminds.webknossos.tracingstore.tracings.skeleton._ -import com.scalableminds.webknossos.tracingstore.tracings.volume.MergedVolumeStats import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingStoreAccessTokenService} -import net.liftweb.common.{Empty, Failure, Full} import play.api.i18n.Messages import play.api.libs.json.Json import com.scalableminds.webknossos.datastore.controllers.Controller @@ -95,38 +93,6 @@ class SkeletonTracingController @Inject()(skeletonTracingService: SkeletonTracin } } - def mergedFromIds(persist: Boolean): Action[List[Option[TracingSelector]]] = - Action.async(validateJson[List[Option[TracingSelector]]]) { implicit request => - log() { - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { - for { - tracingOpts <- annotationService.findMultipleSkeletons(request.body, applyUpdates = true) ?~> Messages( - "tracing.notFound") - tracingsWithIds = tracingOpts.zip(request.body).flatMap { - case (Some(tracing), Some(selector)) => Some((tracing, selector.tracingId)) - case _ => None - } - newTracingId = TracingId.generate - mergedVolumeStats <- skeletonTracingService.mergeVolumeData(request.body.flatten, - tracingsWithIds.map(_._1), - newTracingId, - newVersion = 0L, - toCache = !persist) - mergeEditableMappingsResultBox <- skeletonTracingService - .mergeEditableMappings(newTracingId, tracingsWithIds) - .futureBox - newEditableMappingIdOpt <- mergeEditableMappingsResultBox match { - case Full(()) => Fox.successful(Some(newTracingId)) - case Empty => Fox.successful(None) - case f: Failure => f.toFox - } - mergedTracing <- Fox.box2Fox(skeletonTracingService.merge(tracingsWithIds.map(_._1))) - _ <- skeletonTracingService.save(mergedTracing, Some(newTracingId), version = 0, toCache = !persist) - } yield Ok(Json.toJson(newTracingId)) - } - } - } - def mergedFromContents(persist: Boolean): Action[SkeletonTracings] = Action.async(validateProto[SkeletonTracings]) { implicit request => log() { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index a4ce7ec82a8..c0c1caf54f3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -8,6 +8,7 @@ import com.scalableminds.webknossos.datastore.Annotation.{ AnnotationLayerTypeProto, AnnotationProto } +import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.controllers.Controller import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayer import com.scalableminds.webknossos.datastore.services.UserAccessRequest @@ -25,6 +26,8 @@ import com.scalableminds.webknossos.tracingstore.annotation.{ } import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService import com.scalableminds.webknossos.tracingstore.tracings.skeleton.SkeletonTracingService +import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeTracingService +import net.liftweb.common.{Empty, Failure, Full} import play.api.i18n.Messages import play.api.libs.json.Json import play.api.mvc.{Action, AnyContent, PlayBodyParsers} @@ -37,6 +40,7 @@ class TSAnnotationController @Inject()( annotationService: TSAnnotationService, annotationTransactionService: AnnotationTransactionService, skeletonTracingService: SkeletonTracingService, + volumeTracingService: VolumeTracingService, tracingDataStore: TracingDataStore)(implicit ec: ExecutionContext, bodyParsers: PlayBodyParsers) extends Controller with KeyValueStoreImplicits { @@ -145,38 +149,52 @@ class TSAnnotationController @Inject()( skeletonLayers = annotations.flatMap( _.annotationLayers.filter(_.`type` == AnnotationLayerTypeProto.Skeleton)) volumeLayers = annotations.flatMap(_.annotationLayers.filter(_.`type` == AnnotationLayerTypeProto.Volume)) - // TODO: Volume + newSkeletonId = TracingId.generate + newVolumeId = TracingId.generate + mergedSkeletonName = allEqual(skeletonLayers.map(_.name)) + .getOrElse(AnnotationLayer.defaultSkeletonLayerName) + mergedVolumeName = allEqual(volumeLayers.map(_.name)).getOrElse(AnnotationLayer.defaultVolumeLayerName) // TODO: Merge updates? if so, iron out reverts? // TODO: Merge editable mappings - /*mergedVolumeStats <- volumeTracingService.mergeVolumeData(request.body.flatten, - tracingsWithIds.map(_._1), - newTracingId, - newVersion = 0L, - toCache = !persist) - mergeEditableMappingsResultBox <- skeletonTracingService - .mergeEditableMappings(newTracingId, tracingsWithIds) + volumeTracings <- annotationService + .findMultipleVolumes(volumeLayers.map { l => + Some(TracingSelector(l.tracingId)) + }, applyUpdates = true) + .map(_.flatten) + mergedVolumeStats <- volumeTracingService.mergeVolumeData(volumeLayers.map(_.tracingId), + volumeTracings, + newVolumeId, + newVersion = 0L, + persist = !persist) + mergeEditableMappingsResultBox <- volumeTracingService + .mergeEditableMappings(newVolumeId, volumeTracings.zip(volumeLayers.map(_.tracingId))) .futureBox newEditableMappingIdOpt <- mergeEditableMappingsResultBox match { - case Full(()) => Fox.successful(Some(newTracingId)) + case Full(()) => Fox.successful(Some(newVolumeId)) case Empty => Fox.successful(None) case f: Failure => f.toFox - }*/ - skeletons <- annotationService.findMultipleSkeletons(skeletonLayers.map { l => - Some(TracingSelector(l.tracingId)) - }, applyUpdates = true) - newSkeletonId = TracingId.generate - newVolumeId = TracingId.generate - mergedSkeletonName = allEqual(skeletonLayers.map(_.name)) - .getOrElse(AnnotationLayer.defaultSkeletonLayerName) - mergedVolumeName = allEqual(volumeLayers.map(_.name)).getOrElse(AnnotationLayer.defaultVolumeLayerName) - mergedSkeletonOpt <- Fox.runIf(skeletons.flatten.nonEmpty)( - skeletonTracingService.merge(skeletons.flatten).toFox) - mergedSkeletonLayerOpt: Option[AnnotationLayerProto] = mergedSkeletonOpt.map( + } + mergedVolumeOpt <- Fox.runIf(volumeTracings.nonEmpty)( + volumeTracingService.merge(volumeTracings, mergedVolumeStats, newEditableMappingIdOpt)) + _ <- Fox.runOptional(mergedVolumeOpt)( + volumeTracingService.save(_, Some(newVolumeId), version = 0, toCache = !persist)) + skeletonTracings <- annotationService + .findMultipleSkeletons(skeletonLayers.map { l => + Some(TracingSelector(l.tracingId)) + }, applyUpdates = true) + .map(_.flatten) + mergedSkeletonOpt <- Fox.runIf(skeletonTracings.nonEmpty)( + skeletonTracingService.merge(skeletonTracings).toFox) + mergedSkeletonLayerOpt = mergedSkeletonOpt.map( _ => AnnotationLayerProto(name = mergedSkeletonName, tracingId = newSkeletonId, `type` = AnnotationLayerTypeProto.Skeleton)) - mergedVolumeLayerOpt: Option[AnnotationLayerProto] = None // TODO + mergedVolumeLayerOpt = mergedVolumeOpt.map( + _ => + AnnotationLayerProto(name = mergedVolumeName, + tracingId = newVolumeId, + `type` = AnnotationLayerTypeProto.Volume)) mergedLayers = Seq(mergedSkeletonLayerOpt, mergedVolumeLayerOpt).flatten firstAnnotation <- annotations.headOption.toFox mergedAnnotation = firstAnnotation.withAnnotationLayers(mergedLayers) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index f3458ab24c9..ea67c15b89c 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -34,14 +34,14 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ VolumeSegmentStatisticsService, VolumeTracingService } -import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingId, TracingSelector} +import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingSelector} import com.scalableminds.webknossos.tracingstore.{ TSRemoteDatastoreClient, TSRemoteWebknossosClient, TracingStoreAccessTokenService, TracingStoreConfig } -import net.liftweb.common.{Empty, Failure, Full} +import net.liftweb.common.Empty import play.api.i18n.Messages import play.api.libs.Files.TemporaryFile import play.api.libs.json.Json @@ -135,39 +135,6 @@ class VolumeTracingController @Inject()( } } - def mergedFromIds(persist: Boolean): Action[List[Option[TracingSelector]]] = - Action.async(validateJson[List[Option[TracingSelector]]]) { implicit request => - log() { - accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { - for { - tracingOpts <- annotationService.findMultipleVolumes(request.body, applyUpdates = true) ?~> Messages( - "tracing.notFound") - tracingsWithIds = tracingOpts.zip(request.body).flatMap { - case (Some(tracing), Some(selector)) => Some((tracing, selector.tracingId)) - case _ => None - } - newTracingId = TracingId.generate - mergedVolumeStats <- volumeTracingService.mergeVolumeData(request.body.flatten, - tracingsWithIds.map(_._1), - newTracingId, - newVersion = 0L, - toCache = !persist) - mergeEditableMappingsResultBox <- volumeTracingService - .mergeEditableMappings(newTracingId, tracingsWithIds) - .futureBox - newEditableMappingIdOpt <- mergeEditableMappingsResultBox match { - case Full(()) => Fox.successful(Some(newTracingId)) - case Empty => Fox.successful(None) - case f: Failure => f.toFox - } - mergedTracing <- Fox.box2Fox( - volumeTracingService.merge(tracingsWithIds.map(_._1), mergedVolumeStats, newEditableMappingIdOpt)) - _ <- volumeTracingService.save(mergedTracing, Some(newTracingId), version = 0, toCache = !persist) - } yield Ok(Json.toJson(newTracingId)) - } - } - } - def initialData(tracingId: String, minMag: Option[Int], maxMag: Option[Int]): Action[AnyContent] = Action.async { implicit request => log() { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala index 9c5dcae0db1..7ac23374aeb 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala @@ -1,13 +1,9 @@ package com.scalableminds.webknossos.tracingstore.tracings -import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingStoreRedisStore} import com.scalableminds.webknossos.tracingstore.tracings.TracingType.TracingType -import com.scalableminds.webknossos.tracingstore.tracings.volume.MergedVolumeStats import com.typesafe.scalalogging.LazyLogging -import net.liftweb.common.Box -import play.api.i18n.MessagesProvider import scalapb.{GeneratedMessage, GeneratedMessageCompanion} import scala.concurrent.ExecutionContext @@ -69,14 +65,4 @@ trait TracingService[T <: GeneratedMessage] } } - def remapTooLargeTreeIds(tracing: T): T = tracing - - def mergeVolumeData(tracingSelectors: Seq[TracingSelector], - tracings: Seq[T], - newId: String, - newVersion: Long, - toCache: Boolean)(implicit mp: MessagesProvider, tc: TokenContext): Fox[MergedVolumeStats] - - def mergeEditableMappings(newTracingId: String, tracingsWithIds: List[(T, String)])( - implicit tc: TokenContext): Fox[Unit] } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala index 9c49183a7c8..ad6c21d13d7 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/SkeletonTracingService.scala @@ -1,18 +1,15 @@ package com.scalableminds.webknossos.tracingstore.tracings.skeleton import com.google.inject.Inject -import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int} -import com.scalableminds.util.tools.{Fox, FoxImplicits} +import com.scalableminds.util.tools.FoxImplicits import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.geometry.NamedBoundingBoxProto import com.scalableminds.webknossos.datastore.helpers.{ProtoGeometryImplicits, SkeletonTracingDefaults} import com.scalableminds.webknossos.datastore.models.datasource.AdditionalAxis import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingStoreRedisStore} import com.scalableminds.webknossos.tracingstore.tracings._ -import com.scalableminds.webknossos.tracingstore.tracings.volume.MergedVolumeStats import net.liftweb.common.{Box, Full} -import play.api.i18n.MessagesProvider import scala.concurrent.ExecutionContext @@ -96,22 +93,12 @@ class SkeletonTracingService @Inject()( ) // Can be removed again when https://github.com/scalableminds/webknossos/issues/5009 is fixed - override def remapTooLargeTreeIds(skeletonTracing: SkeletonTracing): SkeletonTracing = + def remapTooLargeTreeIds(skeletonTracing: SkeletonTracing): SkeletonTracing = if (skeletonTracing.trees.exists(_.treeId > 1048576)) { val newTrees = for ((tree, index) <- skeletonTracing.trees.zipWithIndex) yield tree.withTreeId(index + 1) skeletonTracing.withTrees(newTrees) } else skeletonTracing - def mergeVolumeData(tracingSelectors: Seq[TracingSelector], - tracings: Seq[SkeletonTracing], - newId: String, - newVersion: Long, - toCache: Boolean)(implicit mp: MessagesProvider, tc: TokenContext): Fox[MergedVolumeStats] = - Fox.successful(MergedVolumeStats.empty()) - def dummyTracing: SkeletonTracing = SkeletonTracingDefaults.createInstance - def mergeEditableMappings(newTracingId: String, tracingsWithIds: List[(SkeletonTracing, String)])( - implicit tc: TokenContext): Fox[Unit] = - Fox.empty } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index be29e9fa621..b4e8aa105b3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -727,24 +727,24 @@ class VolumeTracingService @Inject()( case (None, None) => None } - private def bucketStreamFromSelector(selector: TracingSelector, tracing: VolumeTracing)( + private def bucketStreamFor(tracingId: String, tracing: VolumeTracing)( implicit tc: TokenContext): Iterator[(BucketPosition, Array[Byte])] = { - val dataLayer = volumeTracingLayer(selector.tracingId, tracing) + val dataLayer = volumeTracingLayer(tracingId, tracing) dataLayer.bucketProvider.bucketStream(Some(tracing.version)) } - def mergeVolumeData(tracingSelectors: Seq[TracingSelector], + def mergeVolumeData(tracingIds: Seq[String], tracings: Seq[VolumeTracing], newId: String, newVersion: Long, - toCache: Boolean)(implicit mp: MessagesProvider, tc: TokenContext): Fox[MergedVolumeStats] = { + persist: Boolean)(implicit mp: MessagesProvider, tc: TokenContext): Fox[MergedVolumeStats] = { val elementClass = tracings.headOption.map(_.elementClass).getOrElse(elementClassToProto(ElementClass.uint8)) val magSets = new mutable.HashSet[Set[Vec3Int]]() - tracingSelectors.zip(tracings).foreach { - case (selector, tracing) => + tracingIds.zip(tracings).foreach { + case (tracingId, tracing) => val magSet = new mutable.HashSet[Vec3Int]() - bucketStreamFromSelector(selector, tracing).foreach { + bucketStreamFor(tracingId, tracing).foreach { case (bucketPosition, _) => magSet.add(bucketPosition.mag) } @@ -770,15 +770,15 @@ class VolumeTracingService @Inject()( val mergedVolume = new MergedVolume(elementClass) - tracingSelectors.zip(tracings).foreach { - case (selector, tracing) => - val bucketStream = bucketStreamFromSelector(selector, tracing) + tracingIds.zip(tracings).foreach { + case (tracingId, tracing) => + val bucketStream = bucketStreamFor(tracingId, tracing) mergedVolume.addLabelSetFromBucketStream(bucketStream, magsIntersection) } - tracingSelectors.zip(tracings).zipWithIndex.foreach { - case ((selector, tracing), sourceVolumeIndex) => - val bucketStream = bucketStreamFromSelector(selector, tracing) + tracingIds.zip(tracings).zipWithIndex.foreach { + case ((tracingIds, tracing), sourceVolumeIndex) => + val bucketStream = bucketStreamFor(tracingIds, tracing) mergedVolume.addFromBucketStream(sourceVolumeIndex, bucketStream, Some(magsIntersection)) } for { @@ -788,9 +788,9 @@ class VolumeTracingService @Inject()( elementClass) mergedAdditionalAxes <- Fox.box2Fox(AdditionalAxis.mergeAndAssertSameAdditionalAxes(tracings.map(t => AdditionalAxis.fromProtosAsOpt(t.additionalAxes)))) - firstTracingSelector <- tracingSelectors.headOption ?~> "merge.noTracings" + firstTracingId <- tracingIds.headOption ?~> "merge.noTracings" firstTracing <- tracings.headOption ?~> "merge.noTracings" - fallbackLayer <- getFallbackLayer(firstTracingSelector.tracingId, firstTracing) + fallbackLayer <- getFallbackLayer(firstTracingId, firstTracing) segmentIndexBuffer = new VolumeSegmentIndexBuffer(newId, volumeSegmentIndexClient, newVersion, @@ -800,7 +800,13 @@ class VolumeTracingService @Inject()( tc) _ <- mergedVolume.withMergedBuckets { (bucketPosition, bucketBytes) => for { - _ <- saveBucket(newId, elementClass, bucketPosition, bucketBytes, newVersion, toCache, mergedAdditionalAxes) + _ <- saveBucket(newId, + elementClass, + bucketPosition, + bucketBytes, + newVersion, + toTemporaryStore = !persist, // TODO unify boolean direction + naming + mergedAdditionalAxes) _ <- Fox.runIf(shouldCreateSegmentIndex)( updateSegmentIndex(segmentIndexBuffer, bucketPosition, diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index d1bb507338b..82274261f1f 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -30,7 +30,6 @@ GET /volume/:tracingId/findData POST /volume/:tracingId/segmentStatistics/volume @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentVolume(tracingId: String) POST /volume/:tracingId/segmentStatistics/boundingBox @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getSegmentBoundingBox(tracingId: String) POST /volume/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.getMultiple -POST /volume/mergedFromIds @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromIds(persist: Boolean) POST /volume/mergedFromContents @com.scalableminds.webknossos.tracingstore.controllers.VolumeTracingController.mergedFromContents(persist: Boolean) # Editable Mappings @@ -72,7 +71,6 @@ GET /volume/zarr3_experimental/:tracingId/:mag/:coordinates POST /skeleton/save @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.save() POST /skeleton/saveMultiple @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.saveMultiple() POST /skeleton/mergedFromContents @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.mergedFromContents(persist: Boolean) -POST /skeleton/mergedFromIds @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.mergedFromIds(persist: Boolean) GET /skeleton/:tracingId @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.get(tracingId: String, version: Option[Long]) POST /skeleton/:tracingId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.duplicate(tracingId: String, editPosition: Option[String], editRotation: Option[String], boundingBox: Option[String]) POST /skeleton/getMultiple @com.scalableminds.webknossos.tracingstore.controllers.SkeletonTracingController.getMultiple From 74c679b47c5fc3490aa636e5437c097c41f608c0 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 30 Oct 2024 11:44:44 +0100 Subject: [PATCH 140/150] small renamings --- .../controllers/SkeletonTracingController.scala | 2 +- .../controllers/TSAnnotationController.scala | 9 ++++----- .../controllers/VolumeTracingController.scala | 2 +- .../tracingstore/tracings/TracingService.scala | 4 ++-- .../tracings/volume/VolumeTracingDownsampling.scala | 2 +- .../tracings/volume/VolumeTracingService.scala | 10 ++++++---- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala index 17893f1670b..0c3bfbc1b1d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala @@ -101,7 +101,7 @@ class SkeletonTracingController @Inject()(skeletonTracingService: SkeletonTracin for { mergedTracing <- Fox.box2Fox(skeletonTracingService.merge(tracings.flatten)) processedTracing = skeletonTracingService.remapTooLargeTreeIds(mergedTracing) - newId <- skeletonTracingService.save(processedTracing, None, processedTracing.version, toCache = !persist) + newId <- skeletonTracingService.save(processedTracing, None, processedTracing.version, toTemporaryStore = !persist) } yield Ok(Json.toJson(newId)) } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index c0c1caf54f3..8c14836fd2d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -8,7 +8,6 @@ import com.scalableminds.webknossos.datastore.Annotation.{ AnnotationLayerTypeProto, AnnotationProto } -import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.controllers.Controller import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayer import com.scalableminds.webknossos.datastore.services.UserAccessRequest @@ -165,9 +164,9 @@ class TSAnnotationController @Inject()( volumeTracings, newVolumeId, newVersion = 0L, - persist = !persist) + persist = persist) mergeEditableMappingsResultBox <- volumeTracingService - .mergeEditableMappings(newVolumeId, volumeTracings.zip(volumeLayers.map(_.tracingId))) + .mergeEditableMappings(newVolumeId, volumeTracings.zip(volumeLayers.map(_.tracingId)), persist) .futureBox newEditableMappingIdOpt <- mergeEditableMappingsResultBox match { case Full(()) => Fox.successful(Some(newVolumeId)) @@ -177,7 +176,7 @@ class TSAnnotationController @Inject()( mergedVolumeOpt <- Fox.runIf(volumeTracings.nonEmpty)( volumeTracingService.merge(volumeTracings, mergedVolumeStats, newEditableMappingIdOpt)) _ <- Fox.runOptional(mergedVolumeOpt)( - volumeTracingService.save(_, Some(newVolumeId), version = 0, toCache = !persist)) + volumeTracingService.save(_, Some(newVolumeId), version = 0, toTemporaryStore = !persist)) skeletonTracings <- annotationService .findMultipleSkeletons(skeletonLayers.map { l => Some(TracingSelector(l.tracingId)) @@ -199,7 +198,7 @@ class TSAnnotationController @Inject()( firstAnnotation <- annotations.headOption.toFox mergedAnnotation = firstAnnotation.withAnnotationLayers(mergedLayers) _ <- Fox.runOptional(mergedSkeletonOpt)( - skeletonTracingService.save(_, Some(newSkeletonId), version = 0L, toCache = !persist)) + skeletonTracingService.save(_, Some(newSkeletonId), version = 0L, toTemporaryStore = !persist)) _ <- tracingDataStore.annotations.put(newAnnotationId, 0L, mergedAnnotation) } yield Ok(mergedAnnotation.toByteArray).as(protobufMimeType) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index ea67c15b89c..665c185a55f 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -168,7 +168,7 @@ class VolumeTracingController @Inject()( // segment lists for multi-volume uploads are not supported yet, compare https://github.com/scalableminds/webknossos/issues/6887 mergedTracing = mt.copy(segments = List.empty) - newId <- volumeTracingService.save(mergedTracing, None, mergedTracing.version, toCache = !persist) + newId <- volumeTracingService.save(mergedTracing, None, mergedTracing.version, toTemporaryStore = !persist) } yield Ok(Json.toJson(newId)) } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala index 7ac23374aeb..c8ac3837bd4 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/TracingService.scala @@ -54,9 +54,9 @@ trait TracingService[T <: GeneratedMessage] } */ - def save(tracing: T, tracingId: Option[String], version: Long, toCache: Boolean = false): Fox[String] = { + def save(tracing: T, tracingId: Option[String], version: Long, toTemporaryStore: Boolean = false): Fox[String] = { val id = tracingId.getOrElse(TracingId.generate) - if (toCache) { + if (toTemporaryStore) { temporaryTracingStore.insert(id, tracing, Some(temporaryStoreTimeout)) temporaryTracingIdStore.insert(temporaryIdKey(id), "", Some(temporaryIdStoreTimeout)) Fox.successful(id) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala index 6e8e80eb1a8..0210da35f84 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala @@ -62,7 +62,7 @@ trait VolumeTracingDownsampling bucket: BucketPosition, data: Array[Byte], version: Long, - toCache: Boolean = false): Fox[Unit] + toTemporaryStore: Boolean = false): Fox[Unit] protected def updateSegmentIndex(segmentIndexBuffer: VolumeSegmentIndexBuffer, bucketPosition: BucketPosition, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index b4e8aa105b3..cf11d610d39 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -598,14 +598,14 @@ class VolumeTracingService @Inject()( def updateMagList(tracingId: String, tracing: VolumeTracing, mags: Set[Vec3Int], - toCache: Boolean = false): Fox[String] = + toTemporaryStore: Boolean = false): Fox[String] = for { _ <- bool2Fox(tracing.version == 0L) ?~> "Tracing has already been edited." _ <- bool2Fox(mags.nonEmpty) ?~> "Mag restrictions result in zero mags" id <- save(tracing.copy(mags = mags.toList.sortBy(_.maxDim).map(vec3IntToProto)), Some(tracingId), tracing.version, - toCache) + toTemporaryStore) } yield id // TODO use or remove @@ -894,10 +894,12 @@ class VolumeTracingService @Inject()( } } - def mergeEditableMappings(newTracingId: String, tracingsWithIds: List[(VolumeTracing, String)])( - implicit tc: TokenContext): Fox[Unit] = + def mergeEditableMappings(newTracingId: String, + tracingsWithIds: List[(VolumeTracing, String)], + persist: Boolean): Fox[Unit] = if (tracingsWithIds.forall(tracingWithId => tracingWithId._1.getHasEditableMapping)) { for { + _ <- bool2Fox(persist) ?~> "Cannot merge editable mappings without “persist” (used by compound annotations)" remoteFallbackLayers <- Fox.serialCombined(tracingsWithIds)(tracingWithId => remoteFallbackLayerFromVolumeTracing(tracingWithId._1, tracingWithId._2)) remoteFallbackLayer <- remoteFallbackLayers.headOption.toFox From 1a59e62786210a1986207256b8dbe615252e9dfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Wed, 30 Oct 2024 17:11:53 +0100 Subject: [PATCH 141/150] always use only data layers sent by tracing store and disregard layers set from core backend --- .../oxalis/model_initialization.ts | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/frontend/javascripts/oxalis/model_initialization.ts b/frontend/javascripts/oxalis/model_initialization.ts index baa43bccb2d..7a022cc4933 100644 --- a/frontend/javascripts/oxalis/model_initialization.ts +++ b/frontend/javascripts/oxalis/model_initialization.ts @@ -138,24 +138,24 @@ export async function initialize( maybeOutdatedAnnotation.tracingStore.url, maybeOutdatedAnnotation.id, ); + const layersWithStats = annotationFromTracingStore.annotationLayers.map((layer) => { + const matchingLayer = maybeOutdatedAnnotation.annotationLayers.find( + (l) => l.tracingId === layer.tracingId, + ); + + return { + tracingId: layer.tracingId, + name: layer.name, + typ: layer.type, + stats: matchingLayer?.stats || {}, + }; + }); const completeAnnotation = { ...maybeOutdatedAnnotation, name: annotationFromTracingStore.name, description: annotationFromTracingStore.description, + annotationLayers: layersWithStats, }; - annotationFromTracingStore.annotationLayers.forEach((layer) => { - if ( - maybeOutdatedAnnotation.annotationLayers.find((l) => l.tracingId === layer.tracingId) == - null - ) { - completeAnnotation.annotationLayers.push({ - tracingId: layer.tracingId, - name: layer.name, - typ: layer.type, - stats: {}, - }); - } - }); annotation = completeAnnotation; } datasetId = { From 0e9fffc428af85b5caa5c3a1570cc33b06d68b98 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 5 Nov 2024 10:55:27 +0100 Subject: [PATCH 142/150] earliestAccessibleVersion; always fetch all layers; wip resetToBase --- app/controllers/AnnotationController.scala | 1 - .../WKRemoteTracingStoreController.scala | 6 +- app/models/annotation/AnnotationService.scala | 14 +- webknossos-datastore/proto/Annotation.proto | 1 + .../TSRemoteWebknossosClient.scala | 20 +- .../annotation/AnnotationReversion.scala | 15 +- .../AnnotationTransactionService.scala | 46 +++- .../annotation/AnnotationUpdateActions.scala | 17 ++ .../annotation/TSAnnotationService.scala | 239 +++++++----------- .../annotation/UpdateActions.scala | 3 + .../EditableMappingController.scala | 23 +- .../controllers/TSAnnotationController.scala | 17 ++ .../controllers/VolumeTracingController.scala | 11 +- .../EditableMappingUpdater.scala | 31 +-- .../volume/VolumeTracingService.scala | 16 +- ...alableminds.webknossos.tracingstore.routes | 1 + 16 files changed, 230 insertions(+), 231 deletions(-) diff --git a/app/controllers/AnnotationController.scala b/app/controllers/AnnotationController.scala index 62b64efea09..b0fbcedabbe 100755 --- a/app/controllers/AnnotationController.scala +++ b/app/controllers/AnnotationController.scala @@ -11,7 +11,6 @@ import com.scalableminds.webknossos.datastore.models.annotation.{ AnnotationLayerType } import com.scalableminds.webknossos.tracingstore.annotation.AnnotationLayerParameters -import com.scalableminds.webknossos.tracingstore.tracings.volume.MagRestrictions import com.scalableminds.webknossos.tracingstore.tracings.{TracingId, TracingType} import mail.{MailchimpClient, MailchimpTag} import models.analytics.{AnalyticsService, CreateAnnotationEvent, OpenAnnotationEvent} diff --git a/app/controllers/WKRemoteTracingStoreController.scala b/app/controllers/WKRemoteTracingStoreController.scala index b659700a52f..4dbef690f7b 100644 --- a/app/controllers/WKRemoteTracingStoreController.scala +++ b/app/controllers/WKRemoteTracingStoreController.scala @@ -8,7 +8,7 @@ import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayer import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId -import com.scalableminds.webknossos.tracingstore.TracingUpdatesReport +import com.scalableminds.webknossos.tracingstore.AnnotationUpdatesReport import com.scalableminds.webknossos.tracingstore.annotation.AnnotationLayerParameters import com.scalableminds.webknossos.tracingstore.tracings.TracingId @@ -88,8 +88,8 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore } yield Ok } - def handleTracingUpdateReport(name: String, key: String): Action[TracingUpdatesReport] = - Action.async(validateJson[TracingUpdatesReport]) { implicit request => + def handleTracingUpdateReport(name: String, key: String): Action[AnnotationUpdatesReport] = + Action.async(validateJson[AnnotationUpdatesReport]) { implicit request => implicit val ctx: DBAccessContext = GlobalAccessContext tracingStoreService.validateAccess(name, key) { _ => val report = request.body diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index 759b8efe180..9f7a48c0aed 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -278,10 +278,13 @@ class AnnotationService @Inject()( AnnotationLayerType.toProto(l.typ) ) } - annotationProto = AnnotationProto(name = Some(AnnotationDefaults.defaultName), - description = Some(AnnotationDefaults.defaultDescription), - version = 0L, - annotationLayers = layersProto) + annotationProto = AnnotationProto( + name = Some(AnnotationDefaults.defaultName), + description = Some(AnnotationDefaults.defaultDescription), + version = 0L, + annotationLayers = layersProto, + earliestAccessibleVersion = 0L + ) _ <- tracingStoreClient.saveAnnotationProto(annotationId, annotationProto) } yield newAnnotationLayers @@ -485,7 +488,8 @@ class AnnotationService @Inject()( name = Some(AnnotationDefaults.defaultName), description = Some(AnnotationDefaults.defaultDescription), version = 0L, - annotationLayers = annotationLayers.map(_.toProto) + annotationLayers = annotationLayers.map(_.toProto), + earliestAccessibleVersion = 0L ) _ <- tracingStoreClient.saveAnnotationProto(annotationBase._id, annotationBaseProto) _ = logger.info(s"inserting base annotation ${annotationBase._id} for task ${task._id}") diff --git a/webknossos-datastore/proto/Annotation.proto b/webknossos-datastore/proto/Annotation.proto index 5eaab3982f7..4aea0922056 100644 --- a/webknossos-datastore/proto/Annotation.proto +++ b/webknossos-datastore/proto/Annotation.proto @@ -12,6 +12,7 @@ message AnnotationProto { optional string description = 2; required int64 version = 3; repeated AnnotationLayerProto annotationLayers = 4; + required int64 earliestAccessibleVersion = 5; } message AnnotationLayerProto { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala index 523daee164b..f500843ebbc 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/TSRemoteWebknossosClient.scala @@ -26,15 +26,15 @@ import play.api.libs.ws.WSResponse import scala.concurrent.ExecutionContext import scala.concurrent.duration.DurationInt -case class TracingUpdatesReport(annotationId: String, - // TODO stats per tracing id? coordinate with frontend - timestamps: List[Instant], - statistics: Option[JsObject], - significantChangesCount: Int, - viewChangesCount: Int, - userToken: Option[String]) -object TracingUpdatesReport { - implicit val jsonFormat: OFormat[TracingUpdatesReport] = Json.format[TracingUpdatesReport] +case class AnnotationUpdatesReport(annotationId: String, + // TODO stats per tracing id? coordinate with frontend + timestamps: List[Instant], + statistics: Option[JsObject], + significantChangesCount: Int, + viewChangesCount: Int, + userToken: Option[String]) +object AnnotationUpdatesReport { + implicit val jsonFormat: OFormat[AnnotationUpdatesReport] = Json.format[AnnotationUpdatesReport] } class TSRemoteWebknossosClient @Inject()( @@ -53,7 +53,7 @@ class TSRemoteWebknossosClient @Inject()( private lazy val annotationIdByTracingIdCache: AlfuCache[String, String] = AlfuCache(maxCapacity = 10000, timeToLive = 5 minutes) - def reportTracingUpdates(tracingUpdatesReport: TracingUpdatesReport): Fox[WSResponse] = + def reportAnnotationUpdates(tracingUpdatesReport: AnnotationUpdatesReport): Fox[WSResponse] = rpc(s"$webknossosUri/api/tracingstores/$tracingStoreName/handleTracingUpdateReport") .addQueryString("key" -> tracingStoreKey) .silent diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala index a251d545fee..6fd848fc7b7 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationReversion.scala @@ -13,7 +13,7 @@ trait AnnotationReversion { def revertDistributedElements(currentAnnotationWithTracings: AnnotationWithTracings, sourceAnnotationWithTracings: AnnotationWithTracings, - revertAction: RevertToVersionAnnotationAction, + sourceVersion: Long, newVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[Unit] = for { _ <- Fox.serialCombined(sourceAnnotationWithTracings.getVolumes) { @@ -22,23 +22,20 @@ trait AnnotationReversion { for { tracingBeforeRevert <- currentAnnotationWithTracings.getVolume(tracingId).toFox _ <- Fox.runIf(!sourceTracing.getHasEditableMapping)( - volumeTracingService.revertVolumeData(tracingId, - revertAction.sourceVersion, - sourceTracing, - newVersion: Long, - tracingBeforeRevert)) + volumeTracingService + .revertVolumeData(tracingId, sourceVersion, sourceTracing, newVersion: Long, tracingBeforeRevert)) _ <- Fox.runIf(sourceTracing.getHasEditableMapping)( - revertEditableMappingFields(currentAnnotationWithTracings, revertAction, tracingId)) + revertEditableMappingFields(currentAnnotationWithTracings, sourceVersion, tracingId)) } yield () } } yield () private def revertEditableMappingFields(currentAnnotationWithTracings: AnnotationWithTracings, - revertAction: RevertToVersionAnnotationAction, + sourceVersion: Long, tracingId: String)(implicit ec: ExecutionContext): Fox[Unit] = for { updater <- currentAnnotationWithTracings.getEditableMappingUpdater(tracingId).toFox - _ <- updater.revertToVersion(revertAction) + _ <- updater.revertToVersion(sourceVersion) _ <- updater.flushBuffersToFossil() } yield () } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala index 2bad1a3e045..6bd9a1e7294 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationTransactionService.scala @@ -1,9 +1,14 @@ package com.scalableminds.webknossos.tracingstore.annotation import com.scalableminds.util.accesscontext.TokenContext +import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.{Fox, JsonHelper} import com.scalableminds.util.tools.Fox.bool2Fox -import com.scalableminds.webknossos.tracingstore.TracingStoreRedisStore +import com.scalableminds.webknossos.tracingstore.{ + TSRemoteWebknossosClient, + TracingStoreRedisStore, + AnnotationUpdatesReport +} import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingDataStore, TracingId} import com.scalableminds.webknossos.tracingstore.tracings.volume.{ BucketMutatingVolumeUpdateAction, @@ -22,6 +27,7 @@ class AnnotationTransactionService @Inject()(handledGroupIdStore: TracingStoreRe uncommittedUpdatesStore: TracingStoreRedisStore, volumeTracingService: VolumeTracingService, tracingDataStore: TracingDataStore, + remoteWebknossosClient: TSRemoteWebknossosClient, annotationService: TSAnnotationService) extends KeyValueStoreImplicits with LazyLogging { @@ -146,6 +152,24 @@ class AnnotationTransactionService @Inject()(handledGroupIdStore: TracingStoreRe ) } + def handleSingleUpdateAction(annotationId: String, currentVersion: Long, updateAction: UpdateAction)( + implicit ec: ExecutionContext, + tc: TokenContext): Fox[Long] = { + val wrapped = List( + UpdateActionGroup( + currentVersion + 1, + System.currentTimeMillis(), + None, + List(updateAction), + None, + None, + "dummyTransactionId", + 1, + 0 + )) + handleUpdateGroups(annotationId, wrapped) + } + def handleUpdateGroups(annotationId: String, updateGroups: List[UpdateActionGroup])(implicit ec: ExecutionContext, tc: TokenContext): Fox[Long] = if (updateGroups.forall(_.transactionGroupCount == 1)) { @@ -161,7 +185,7 @@ class AnnotationTransactionService @Inject()(handledGroupIdStore: TracingStoreRe private def commitUpdates(annotationId: String, updateGroups: List[UpdateActionGroup])(implicit ec: ExecutionContext, tc: TokenContext): Fox[Long] = for { - _ <- annotationService.reportUpdates(annotationId, updateGroups) + _ <- reportUpdates(annotationId, updateGroups) currentCommittedVersion: Fox[Long] = annotationService.currentMaterializableVersion(annotationId) _ = logger.info(s"trying to commit ${updateGroups .map(_.actions.length) @@ -228,9 +252,9 @@ class AnnotationTransactionService @Inject()(handledGroupIdStore: TracingStoreRe case first :: rest => first.addInfo(updateActionGroup.info) :: rest } actionsWithInfo.map { - case a: UpdateBucketVolumeAction => a.withoutBase64Data + case a: UpdateBucketVolumeAction => a.withoutBase64Data case a: AddLayerAnnotationAction => a.copy(tracingId = Some(TracingId.generate)) - case a => a + case a => a } } @@ -250,4 +274,18 @@ class AnnotationTransactionService @Inject()(handledGroupIdStore: TracingStoreRe } yield updateGroup.version } + private def reportUpdates(annotationId: String, updateGroups: List[UpdateActionGroup])( + implicit tc: TokenContext): Fox[Unit] = + for { + _ <- remoteWebknossosClient.reportAnnotationUpdates( + AnnotationUpdatesReport( + annotationId, + timestamps = updateGroups.map(g => Instant(g.timestamp)), + statistics = updateGroups.flatMap(_.stats).lastOption, // TODO statistics per tracing/layer + significantChangesCount = updateGroups.map(_.significantChangesCount).sum, + viewChangesCount = updateGroups.map(_.viewChangesCount).sum, + tc.userTokenOpt + )) + } yield () + } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala index 95a35cd4c95..5dc0eb5e273 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/AnnotationUpdateActions.scala @@ -93,6 +93,19 @@ case class RevertToVersionAnnotationAction(sourceVersion: Long, this.copy(actionAuthorId = authorId) } +// Used only in tasks by admin to undo the work done of the annotator +case class ResetToBaseAnnotationAction(actionTimestamp: Option[Long] = None, + actionAuthorId: Option[String] = None, + info: Option[String] = None) + extends AnnotationUpdateAction + with ApplyImmediatelyUpdateAction { + override def addTimestamp(timestamp: Long): UpdateAction = + this.copy(actionTimestamp = Some(timestamp)) + override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def addAuthorId(authorId: Option[String]): UpdateAction = + this.copy(actionAuthorId = authorId) +} + case class UpdateTdCameraAnnotationAction(actionTimestamp: Option[Long] = None, actionAuthorId: Option[String] = None, info: Option[String] = None) @@ -125,6 +138,10 @@ object RevertToVersionAnnotationAction { implicit val jsonFormat: OFormat[RevertToVersionAnnotationAction] = Json.format[RevertToVersionAnnotationAction] } +object ResetToBaseAnnotationAction { + implicit val jsonFormat: OFormat[ResetToBaseAnnotationAction] = + Json.format[ResetToBaseAnnotationAction] +} object UpdateTdCameraAnnotationAction { implicit val jsonFormat: OFormat[UpdateTdCameraAnnotationAction] = Json.format[UpdateTdCameraAnnotationAction] } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 9bf7ff95522..2b020c5abdc 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -3,7 +3,6 @@ package com.scalableminds.webknossos.tracingstore.annotation import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.cache.AlfuCache import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int} -import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox import com.scalableminds.util.tools.Fox.{bool2Fox, box2Fox, option2Fox} import com.scalableminds.webknossos.datastore.Annotation.{ @@ -35,7 +34,6 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{ MagRestrictions, UpdateMappingNameVolumeAction, VolumeTracingService, - VolumeUpdateAction } import com.scalableminds.webknossos.tracingstore.tracings.{ FallbackDataHelper, @@ -45,11 +43,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.{ TracingSelector, VersionedKeyValuePair } -import com.scalableminds.webknossos.tracingstore.{ - TSRemoteDatastoreClient, - TSRemoteWebknossosClient, - TracingUpdatesReport -} +import com.scalableminds.webknossos.tracingstore.{TSRemoteDatastoreClient, TSRemoteWebknossosClient} import com.typesafe.scalalogging.LazyLogging import net.liftweb.common.{Empty, Full} import play.api.libs.json.{JsObject, JsValue, Json} @@ -71,22 +65,46 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss with LazyLogging { private lazy val materializedAnnotationWithTracingCache = - // annotation id, version, requestedSkeletons, requestedVolumes, requestAll - // TODO instead of requested, use list of tracings determined from requests + updates? - AlfuCache[(String, Long, List[String], List[String], Boolean), AnnotationWithTracings](maxCapacity = 1000) + // annotation id, version + AlfuCache[(String, Long), AnnotationWithTracings](maxCapacity = 1000) - def reportUpdates(annotationId: String, updateGroups: List[UpdateActionGroup])(implicit tc: TokenContext): Fox[Unit] = + def get(annotationId: String, version: Option[Long])(implicit ec: ExecutionContext, + tc: TokenContext): Fox[AnnotationProto] = for { - _ <- remoteWebknossosClient.reportTracingUpdates( - TracingUpdatesReport( - annotationId, - timestamps = updateGroups.map(g => Instant(g.timestamp)), - statistics = updateGroups.flatMap(_.stats).lastOption, // TODO statistics per tracing/layer - significantChangesCount = updateGroups.map(_.significantChangesCount).sum, - viewChangesCount = updateGroups.map(_.viewChangesCount).sum, - tc.userTokenOpt - )) - } yield () + withTracings <- getWithTracings(annotationId, version) + } yield withTracings.annotation + + def getMultiple(annotationIds: Seq[String])(implicit ec: ExecutionContext, + tc: TokenContext): Fox[Seq[AnnotationProto]] = + Fox.serialCombined(annotationIds) { annotationId => + get(annotationId, None) + } + + private def getWithTracings(annotationId: String, version: Option[Long])( + implicit ec: ExecutionContext, + tc: TokenContext): Fox[AnnotationWithTracings] = + for { + newestMaterialized <- getNewestMaterialized(annotationId) + targetVersion <- determineTargetVersion(annotationId, newestMaterialized, version) ?~> "determineTargetVersion.failed" + // When requesting any other than the newest version, do not consider the changes final + reportChangesToWk = version.isEmpty || version.contains(targetVersion) + updatedAnnotation <- materializedAnnotationWithTracingCache.getOrLoad( + (annotationId, targetVersion), + _ => getWithTracingsVersioned(annotationId, targetVersion, reportChangesToWk = reportChangesToWk) + ) + } yield updatedAnnotation + + private def getWithTracingsVersioned(annotationId: String, version: Long, reportChangesToWk: Boolean)( + implicit ec: ExecutionContext, + tc: TokenContext): Fox[AnnotationWithTracings] = + for { + annotationWithVersion <- tracingDataStore.annotations.get(annotationId, Some(version))( + fromProtoBytes[AnnotationProto]) ?~> "getAnnotation.failed" + _ = logger.info( + s"cache miss for $annotationId v$version, applying updates from ${annotationWithVersion.version} to $version...") + annotation = annotationWithVersion.value + updated <- applyPendingUpdates(annotation, annotationId, version, reportChangesToWk) ?~> "applyUpdates.failed" + } yield updated def currentMaterializableVersion(annotationId: String): Fox[Long] = tracingDataStore.annotationUpdates.getVersion(annotationId, mayBeEmpty = Some(true), emptyFallback = Some(0L)) @@ -94,6 +112,12 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss def currentMaterializedVersion(annotationId: String): Fox[Long] = tracingDataStore.annotations.getVersion(annotationId, mayBeEmpty = Some(true), emptyFallback = Some(0L)) + private def getNewestMaterialized(annotationId: String): Fox[AnnotationProto] = + for { + keyValuePair <- tracingDataStore.annotations.get[AnnotationProto](annotationId, mayBeEmpty = Some(true))( + fromProtoBytes[AnnotationProto]) ?~> "getAnnotation.failed" + } yield keyValuePair.value + private def findPendingUpdates(annotationId: String, existingVersion: Long, desiredVersion: Long)( implicit ec: ExecutionContext): Fox[List[(Long, List[UpdateAction])]] = if (desiredVersion == existingVersion) Fox.successful(List()) @@ -133,7 +157,9 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss case a: EditableMappingUpdateAction => annotationWithTracings.applyEditableMappingAction(a) case a: RevertToVersionAnnotationAction => - revertToVersion(annotationId, annotationWithTracings, a, targetVersion) // TODO if the revert action is not isolated, we need not the target version of all but the target version of this update + revertToVersion(annotationId, annotationWithTracings, a, targetVersion) // TODO: if the revert action is not isolated, we need not the target version of all but the target version of this update + case _: ResetToBaseAnnotationAction => + resetToBase(annotationId, annotationWithTracings, targetVersion) case _: BucketMutatingVolumeUpdateAction => Fox.successful(annotationWithTracings) // No-op, as bucket-mutating actions are performed eagerly, so not here. case _ => Fox.failure(s"Received unsupported AnnotationUpdateAction action ${Json.toJson(updateAction)}") @@ -163,19 +189,27 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss annotationWithTracings: AnnotationWithTracings, revertAction: RevertToVersionAnnotationAction, newVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = - // Note: works only after “ironing out” the update action groups + // Note: works only if revert actions are in separate update groups for { - sourceAnnotation: AnnotationWithTracings <- getWithTracings( - annotationId, - Some(revertAction.sourceVersion), - List.empty, - List.empty, - requestAll = true) // TODO do we need to request the others? + _ <- bool2Fox(revertAction.sourceVersion >= annotationWithTracings.annotation.earliestAccessibleVersion) ?~> f"Trying to revert to ${revertAction.sourceVersion}, but earliest accessible is ${annotationWithTracings.annotation.earliestAccessibleVersion}" + sourceAnnotation: AnnotationWithTracings <- getWithTracings(annotationId, Some(revertAction.sourceVersion)) _ = logger.info( s"reverting to suorceVersion ${revertAction.sourceVersion}. got sourceAnnotation with version ${sourceAnnotation.version} with ${sourceAnnotation.skeletonStats}") - _ <- revertDistributedElements(annotationWithTracings, sourceAnnotation, revertAction, newVersion) + _ <- revertDistributedElements(annotationWithTracings, sourceAnnotation, revertAction.sourceVersion, newVersion) } yield sourceAnnotation + private def resetToBase(annotationId: String, annotationWithTracings: AnnotationWithTracings, newVersion: Long)( + implicit ec: ExecutionContext, + tc: TokenContext): Fox[AnnotationWithTracings] = { + // Note: works only if reset actions are in separate update groups + val sourceVersion = 0L // Tasks are always created with as v0 currently + logger.info(s"Resetting annotation $annotationId to base (v$sourceVersion)") + for { + sourceAnnotation: AnnotationWithTracings <- getWithTracings(annotationId, Some(sourceVersion)) + _ <- revertDistributedElements(annotationWithTracings, sourceAnnotation, sourceVersion, newVersion) + } yield sourceAnnotation + } + def updateActionLog(annotationId: String, newestVersion: Long, oldestVersion: Long)( implicit ec: ExecutionContext): Fox[JsValue] = { def versionedTupleToJson(tuple: (Long, List[UpdateAction])): JsObject = @@ -197,69 +231,11 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } yield Json.toJson(updateActionBatches.flatten.map(versionedTupleToJson)) } - def get(annotationId: String, version: Option[Long])(implicit ec: ExecutionContext, - tc: TokenContext): Fox[AnnotationProto] = - for { - withTracings <- getWithTracings(annotationId, version, List.empty, List.empty, requestAll = false) - } yield withTracings.annotation - - def getMultiple(annotationIds: Seq[String])(implicit ec: ExecutionContext, - tc: TokenContext): Fox[Seq[AnnotationProto]] = - Fox.serialCombined(annotationIds) { annotationId => - get(annotationId, None) - } - - private def getWithTracings( - annotationId: String, - version: Option[Long], - requestedSkeletonTracingIds: List[String], - requestedVolumeTracingIds: List[String], - requestAll: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = - for { - targetVersion <- determineTargetVersion(annotationId, version) ?~> "determineTargetVersion.failed" - // When requesting any other than the newest version, do not consider the changes final - reportChangesToWk = version.isEmpty || version.contains(targetVersion) - updatedAnnotation <- materializedAnnotationWithTracingCache.getOrLoad( - (annotationId, targetVersion, requestedSkeletonTracingIds, requestedVolumeTracingIds, requestAll), - _ => - getWithTracingsVersioned( - annotationId, - targetVersion, - requestedSkeletonTracingIds, - requestedVolumeTracingIds, - requestAll = true, - reportChangesToWk = reportChangesToWk) // TODO can we request fewer to save perf? still need to avoid duplicate apply - ) - } yield updatedAnnotation - - private def getWithTracingsVersioned( - annotationId: String, - version: Long, - requestedSkeletonTracingIds: List[String], - requestedVolumeTracingIds: List[String], - requestAll: Boolean, - reportChangesToWk: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = - for { - annotationWithVersion <- tracingDataStore.annotations.get(annotationId, Some(version))( - fromProtoBytes[AnnotationProto]) ?~> "getAnnotation.failed" - _ = logger.info( - s"cache miss for ${annotationId} v$version, requested ${requestedSkeletonTracingIds.mkString(",")} + ${requestedVolumeTracingIds - .mkString(",")} (requestAll=$requestAll). Applying updates from ${annotationWithVersion.version} to $version...") - annotation = annotationWithVersion.value - updated <- applyPendingUpdates(annotation, - annotationId, - version, - requestedSkeletonTracingIds, - requestedVolumeTracingIds, - requestAll, - reportChangesToWk) ?~> "applyUpdates.failed" - } yield updated - def findEditableMappingInfo(annotationId: String, tracingId: String, version: Option[Long] = None)( implicit ec: ExecutionContext, tc: TokenContext): Fox[EditableMappingInfo] = for { - annotation <- getWithTracings(annotationId, version, List.empty, List(tracingId), requestAll = false) ?~> "getWithTracings.failed" + annotation <- getWithTracings(annotationId, version) ?~> "getWithTracings.failed" tracing <- annotation.getEditableMappingInfo(tracingId) ?~> "getEditableMapping.failed" } yield tracing @@ -284,23 +260,14 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss annotation: AnnotationProto, annotationId: String, targetVersion: Long, - requestedSkeletonTracingIds: List[String], - requestedVolumeTracingIds: List[String], - requestAll: Boolean, reportChangesToWk: Boolean)(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationWithTracings] = for { updateGroupsAsSaved <- findPendingUpdates(annotationId, annotation.version, targetVersion) ?~> "findPendingUpdates.failed" updatesGroupsRegrouped = regroupByIsolationSensitiveActions(updateGroupsAsSaved) - updatesFlat = updatesGroupsRegrouped.flatMap(_._2) - annotationWithTracings <- findTracingsForUpdates(annotation, - updatesFlat, - requestedSkeletonTracingIds, - requestedVolumeTracingIds, - requestAll) ?~> "findTracingsForUpdates.failed" - annotationWithTracingsAndMappings <- findEditableMappingsForUpdates( + annotationWithTracings <- findTracingsForAnnotation(annotation) ?~> "findTracingsForUpdates.failed" + annotationWithTracingsAndMappings <- findEditableMappingsForAnnotation( annotationId, annotationWithTracings, - updatesFlat, annotation.version, targetVersion) // TODO: targetVersion must be set per update group, as reverts may come between these updated <- applyUpdatesGrouped(annotationWithTracingsAndMappings, @@ -309,15 +276,13 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss reportChangesToWk) ?~> "applyUpdates.inner.failed" } yield updated - private def findEditableMappingsForUpdates( // TODO integrate with findTracings? - annotationId: String, - annotationWithTracings: AnnotationWithTracings, - updates: List[UpdateAction], - currentMaterializedVersion: Long, - targetVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext) = { + private def findEditableMappingsForAnnotation( + annotationId: String, + annotationWithTracings: AnnotationWithTracings, + currentMaterializedVersion: Long, + targetVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext) = { val volumeWithEditableMapping = annotationWithTracings.volumesThatHaveEditableMapping logger.info(s"fetching editable mappings ${volumeWithEditableMapping.map(_._2).mkString(",")}") - // TODO perf optimization: intersect with editable mapping updates? unless requested for { idInfoUpdaterTuples <- Fox.serialCombined(volumeWithEditableMapping) { case (volumeTracing, volumeTracingId) => @@ -364,30 +329,13 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss relyOnAgglomerateIds = false // TODO should we? ) - private def findTracingsForUpdates( - annotation: AnnotationProto, - updates: List[UpdateAction], - requestedSkeletonTracingIds: List[String], - requestedVolumeTracingIds: List[String], - requestAll: Boolean)(implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = { + private def findTracingsForAnnotation(annotation: AnnotationProto)( + implicit ec: ExecutionContext): Fox[AnnotationWithTracings] = { val skeletonTracingIds = - if (requestAll) - annotation.annotationLayers.filter(_.`type` == AnnotationLayerTypeProto.Skeleton).map(_.tracingId) - else { - (updates.flatMap { - case u: SkeletonUpdateAction => Some(u.actionTracingId) - case _ => None - } ++ requestedSkeletonTracingIds).distinct - } + annotation.annotationLayers.filter(_.`type` == AnnotationLayerTypeProto.Skeleton).map(_.tracingId) + val volumeTracingIds = - if (requestAll) - annotation.annotationLayers.filter(_.`type` == AnnotationLayerTypeProto.Volume).map(_.tracingId) - else { - (updates.flatMap { - case u: VolumeUpdateAction => Some(u.actionTracingId) - case _ => None - } ++ requestedVolumeTracingIds).distinct - } + annotation.annotationLayers.filter(_.`type` == AnnotationLayerTypeProto.Volume).map(_.tracingId) logger.info(s"fetching volumes $volumeTracingIds and skeletons $skeletonTracingIds") for { @@ -494,22 +442,26 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss private def flushAnnotationInfo(annotationId: String, annotationWithTracings: AnnotationWithTracings) = tracingDataStore.annotations.put(annotationId, annotationWithTracings.version, annotationWithTracings.annotation) - private def determineTargetVersion(annotationId: String, targetVersionOpt: Option[Long]): Fox[Long] = + private def determineTargetVersion(annotationId: String, + newestMaterializedAnnotation: AnnotationProto, + requestedVersionOpt: Option[Long]): Fox[Long] = /* * Determines the newest saved version from the updates column. * if there are no updates at all, assume annotation is brand new (possibly created from NML, - * hence the emptyFallbck annotation.version) + * hence the emptyFallbck newestMaterializedAnnotation.version) */ for { newestUpdateVersion <- tracingDataStore.annotationUpdates.getVersion(annotationId, mayBeEmpty = Some(true), - emptyFallback = Some(0L)) - } yield { - targetVersionOpt match { - case None => newestUpdateVersion - case Some(desiredSome) => math.min(desiredSome, newestUpdateVersion) + emptyFallback = + Some(newestMaterializedAnnotation.version)) + targetVersion = requestedVersionOpt match { + case None => newestUpdateVersion + case Some(requestedVersion) => + math.max(newestMaterializedAnnotation.earliestAccessibleVersion, + math.min(requestedVersion, newestUpdateVersion)) } - } + } yield targetVersion def updateActionStatistics(tracingId: String): Fox[JsObject] = for { @@ -568,10 +520,10 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss def findVolume(annotationId: String, tracingId: String, version: Option[Long] = None, - useCache: Boolean = true, + useCache: Boolean = true, // TODO applyUpdates: Boolean = false)(implicit tc: TokenContext, ec: ExecutionContext): Fox[VolumeTracing] = for { - annotation <- getWithTracings(annotationId, version, List.empty, List(tracingId), requestAll = false) // TODO is applyUpdates still needed? + annotation <- getWithTracings(annotationId, version) // TODO is applyUpdates still needed? tracing <- annotation.getVolume(tracingId) } yield tracing @@ -579,13 +531,13 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss annotationId: String, tracingId: String, version: Option[Long] = None, - useCache: Boolean = true, + useCache: Boolean = true, // TODO applyUpdates: Boolean = false)(implicit tc: TokenContext, ec: ExecutionContext): Fox[SkeletonTracing] = if (tracingId == TracingId.dummy) Fox.successful(skeletonTracingService.dummyTracing) else { for { - annotation <- getWithTracings(annotationId, version, List(tracingId), List.empty, requestAll = false) // TODO is applyUpdates still needed? + annotation <- getWithTracings(annotationId, version) // TODO is applyUpdates still needed? tracing <- annotation.getSkeleton(tracingId) } yield tracing } @@ -635,13 +587,14 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss newLayers <- Fox.serialCombined(currentAnnotation.annotationLayers)(layer => duplicateLayer(annotationId, layer, currentAnnotation.version, isFromTask, datasetBoundingBox)) _ <- duplicateUpdates(annotationId, newAnnotationId) - duplicatedAnnotation = currentAnnotation.copy(annotationLayers = newLayers) + duplicatedAnnotation = currentAnnotation.copy(annotationLayers = newLayers, + earliestAccessibleVersion = currentAnnotation.version) _ <- tracingDataStore.annotations.put(newAnnotationId, currentAnnotation.version, duplicatedAnnotation) } yield duplicatedAnnotation private def duplicateUpdates(annotationId: String, newAnnotationId: String)( implicit ec: ExecutionContext): Fox[Unit] = - // TODO perf: batch or use fossildb duplicate api + // TODO memory: batch for { updatesAsBytes: Seq[(Long, Array[Byte])] <- tracingDataStore.annotationUpdates .getMultipleVersionsAsVersionValueTuple(annotationId) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala index 091ae7f327e..c81be809cb7 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala @@ -110,6 +110,7 @@ object UpdateAction { case "updateLayerMetadata" => deserialize[UpdateLayerMetadataAnnotationAction](jsonValue) case "updateMetadataOfAnnotation" => deserialize[UpdateMetadataAnnotationAction](jsonValue) case "revertToVersion" => deserialize[RevertToVersionAnnotationAction](jsonValue) + case "resetToBase" => deserialize[ResetToBaseAnnotationAction](jsonValue) case "updateTdCamera" => deserialize[UpdateTdCameraAnnotationAction](jsonValue) case unknownAction: String => JsError(s"Invalid update action s'$unknownAction'") @@ -215,6 +216,8 @@ object UpdateAction { "value" -> Json.toJson(s)(UpdateMetadataAnnotationAction.jsonFormat)) case s: RevertToVersionAnnotationAction => Json.obj("name" -> "revertToVersion", "value" -> Json.toJson(s)(RevertToVersionAnnotationAction.jsonFormat)) + case s: ResetToBaseAnnotationAction => + Json.obj("name" -> "resetToBase", "value" -> Json.toJson(s)(ResetToBaseAnnotationAction.jsonFormat)) case s: UpdateTdCameraAnnotationAction => Json.obj("name" -> "updateTdCamera", "value" -> Json.toJson(s)(UpdateTdCameraAnnotationAction.jsonFormat)) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala index 5abf9014d24..a1e6fc9a3b7 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/EditableMappingController.scala @@ -8,11 +8,7 @@ import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing import com.scalableminds.webknossos.datastore.controllers.Controller import com.scalableminds.webknossos.datastore.services.{EditableMappingSegmentListResult, UserAccessRequest} import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingStoreAccessTokenService} -import com.scalableminds.webknossos.tracingstore.annotation.{ - AnnotationTransactionService, - TSAnnotationService, - UpdateActionGroup -} +import com.scalableminds.webknossos.tracingstore.annotation.{AnnotationTransactionService, TSAnnotationService} import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.{ EditableMappingService, MinCutParameters, @@ -52,19 +48,10 @@ class EditableMappingController @Inject()(volumeTracingService: VolumeTracingSer actionTracingId = tracingId, actionTimestamp = Some(System.currentTimeMillis())) _ <- annotationTransactionService - .handleUpdateGroups( // TODO replace this route by the update action only? address editable mappings by volume tracing id? - annotationId, - List( - UpdateActionGroup(tracing.version + 1, - System.currentTimeMillis(), - None, - List(volumeUpdate), - None, - None, - "dummyTransactionId", - 1, - 0)) - ) + .handleSingleUpdateAction( // TODO replace this route by the update action only? + annotationId, + tracing.version, + volumeUpdate) infoJson = editableMappingService.infoJson(tracingId = tracingId, editableMappingInfo = editableMappingInfo) } yield Ok(infoJson) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index 8c14836fd2d..2f543b04548 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -20,6 +20,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.{ import com.scalableminds.webknossos.tracingstore.TracingStoreAccessTokenService import com.scalableminds.webknossos.tracingstore.annotation.{ AnnotationTransactionService, + ResetToBaseAnnotationAction, TSAnnotationService, UpdateActionGroup } @@ -138,6 +139,22 @@ class TSAnnotationController @Inject()( } } + def resetToBase(annotationId: String): Action[AnyContent] = + Action.async { implicit request => + log() { + logTime(slackNotificationService.noticeSlowRequest) { + accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { + for { + currentVersion <- annotationService.currentMaterializableVersion(annotationId) + _ <- annotationTransactionService.handleSingleUpdateAction(annotationId, + currentVersion, + ResetToBaseAnnotationAction()) + } yield Ok + } + } + } + } + def mergedFromIds(persist: Boolean, newAnnotationId: String): Action[List[String]] = Action.async(validateJson[List[String]]) { implicit request => log() { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index 665c185a55f..c7444ae982d 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -26,6 +26,7 @@ import com.scalableminds.webknossos.tracingstore.annotation.{AnnotationTransacti import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService import com.scalableminds.webknossos.tracingstore.tracings.volume.{ + ImportVolumeDataVolumeAction, MagRestrictions, MergedVolumeStats, TSFullMeshService, @@ -249,11 +250,11 @@ class VolumeTracingController @Inject()( tracing <- annotationService.findVolume(annotationId, tracingId) ?~> Messages("tracing.notFound") currentVersion <- request.body.dataParts("currentVersion").headOption.flatMap(_.toIntOpt).toFox zipFile <- request.body.files.headOption.map(f => new File(f.ref.path.toString)).toFox - (updateGroup, largestSegmentId) <- volumeTracingService.importVolumeData(tracingId, - tracing, - zipFile, - currentVersion) - _ <- annotationTransactionService.handleUpdateGroups(annotationId, List(updateGroup)) + largestSegmentId <- volumeTracingService.importVolumeData(tracingId, tracing, zipFile, currentVersion) + _ <- annotationTransactionService.handleSingleUpdateAction( + annotationId, + tracing.version, + ImportVolumeDataVolumeAction(tracingId, Some(largestSegmentId))) } yield Ok(Json.toJson(largestSegmentId)) } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala index d7231195a67..fc545c9e8d7 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdater.scala @@ -10,11 +10,7 @@ import com.scalableminds.webknossos.datastore.SegmentToAgglomerateProto.{ SegmentToAgglomerateChunkProto } import com.scalableminds.webknossos.tracingstore.TSRemoteDatastoreClient -import com.scalableminds.webknossos.tracingstore.annotation.{ - RevertToVersionAnnotationAction, - TSAnnotationService, - UpdateAction -} +import com.scalableminds.webknossos.tracingstore.annotation.{TSAnnotationService, UpdateAction} import com.scalableminds.webknossos.tracingstore.tracings.volume.ReversionHelper import com.scalableminds.webknossos.tracingstore.tracings.{ KeyValueStoreImplicits, @@ -418,9 +414,9 @@ class EditableMappingUpdater( ) } - def revertToVersion(revertAction: RevertToVersionAnnotationAction)(implicit ec: ExecutionContext): Fox[Unit] = + def revertToVersion(sourceVersion: Long)(implicit ec: ExecutionContext): Fox[Unit] = for { - _ <- bool2Fox(revertAction.sourceVersion <= oldVersion) ?~> "trying to revert editable mapping to a version not yet present in the database" + _ <- bool2Fox(sourceVersion <= oldVersion) ?~> "trying to revert editable mapping to a version not yet present in the database" _ = segmentToAgglomerateBuffer.clear() _ = agglomerateToGraphBuffer.clear() segmentToAgglomerateChunkNewestStream = new VersionedSegmentToAgglomerateChunkIterator( @@ -428,16 +424,13 @@ class EditableMappingUpdater( tracingDataStore.editableMappingsSegmentToAgglomerate) _ <- Fox.serialCombined(segmentToAgglomerateChunkNewestStream) { case (chunkKey, _, version) => - if (version > revertAction.sourceVersion) { - editableMappingService - .getSegmentToAgglomerateChunk(chunkKey, Some(revertAction.sourceVersion)) - .futureBox - .map { - case Full(chunkData) => segmentToAgglomerateBuffer.put(chunkKey, (chunkData.toMap, false)) - case Empty => segmentToAgglomerateBuffer.put(chunkKey, (Map[Long, Long](), true)) - case Failure(msg, _, chain) => - Fox.failure(msg, Empty, chain) - } + if (version > sourceVersion) { + editableMappingService.getSegmentToAgglomerateChunk(chunkKey, Some(sourceVersion)).futureBox.map { + case Full(chunkData) => segmentToAgglomerateBuffer.put(chunkKey, (chunkData.toMap, false)) + case Empty => segmentToAgglomerateBuffer.put(chunkKey, (Map[Long, Long](), true)) + case Failure(msg, _, chain) => + Fox.failure(msg, Empty, chain) + } } else Fox.successful(()) } agglomerateToGraphNewestStream = new VersionedAgglomerateToGraphIterator( @@ -445,11 +438,11 @@ class EditableMappingUpdater( tracingDataStore.editableMappingsAgglomerateToGraph) _ <- Fox.serialCombined(agglomerateToGraphNewestStream) { case (graphKey, _, version) => - if (version > revertAction.sourceVersion) { + if (version > sourceVersion) { for { agglomerateId <- agglomerateIdFromAgglomerateGraphKey(graphKey) _ <- editableMappingService - .getAgglomerateGraphForId(tracingId, revertAction.sourceVersion, agglomerateId) + .getAgglomerateGraphForId(tracingId, sourceVersion, agglomerateId) .futureBox .map { case Full(graphData) => agglomerateToGraphBuffer.put(graphKey, (graphData, false)) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index cf11d610d39..bd8812f36a5 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -23,7 +23,6 @@ import com.scalableminds.webknossos.datastore.models.{ WebknossosAdHocMeshRequest } import com.scalableminds.webknossos.datastore.services._ -import com.scalableminds.webknossos.tracingstore.annotation.UpdateActionGroup import com.scalableminds.webknossos.tracingstore.tracings.TracingType.TracingType import com.scalableminds.webknossos.tracingstore.tracings._ import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeDataZipFormat.VolumeDataZipFormat @@ -824,7 +823,7 @@ class VolumeTracingService @Inject()( def importVolumeData(tracingId: String, tracing: VolumeTracing, zipFile: File, currentVersion: Int)( implicit mp: MessagesProvider, - tc: TokenContext): Fox[(UpdateActionGroup, Long)] = + tc: TokenContext): Fox[Long] = if (currentVersion != tracing.version) Fox.failure("version.mismatch") else { @@ -879,18 +878,7 @@ class VolumeTracingService @Inject()( } yield () } _ <- segmentIndexBuffer.flush() - updateGroup = UpdateActionGroup( - tracing.version + 1, - System.currentTimeMillis(), - None, - List(ImportVolumeDataVolumeAction(tracingId, Some(mergedVolume.largestSegmentId.toPositiveLong))), - None, - None, - "dummyTransactionId", - 1, - 0 - ) - } yield (updateGroup, mergedVolume.largestSegmentId.toPositiveLong) + } yield mergedVolume.largestSegmentId.toPositiveLong } } diff --git a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes index 82274261f1f..5bb6a936629 100644 --- a/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes +++ b/webknossos-tracingstore/conf/com.scalableminds.webknossos.tracingstore.routes @@ -12,6 +12,7 @@ GET /annotation/:annotationId/updateActionLog GET /annotation/:annotationId/updateActionStatistics @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionStatistics(annotationId: String) GET /annotation/:annotationId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.newestVersion(annotationId: String) POST /annotation/:annotationId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.duplicate(annotationId: String, newAnnotationId: String, version: Option[Long], isFromTask: Boolean, datasetBoundingBox: Option[String]) +POST /annotation/:annotationId/resetToBase @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.resetToBase(annotationId: String) POST /annotation/mergedFromIds @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.mergedFromIds(persist: Boolean, newAnnotationId: String) # Volume tracings From 02f78b766341f5b59103de52e3fbaf7a1b153470 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 5 Nov 2024 11:11:46 +0100 Subject: [PATCH 143/150] call resetToBase --- app/controllers/AnnotationIOController.scala | 4 +- app/models/annotation/AnnotationService.scala | 56 +++---------------- .../WKRemoteTracingStoreClient.scala | 7 +++ ...sampling.scala => VolumeTracingMags.scala} | 6 +- .../volume/VolumeTracingService.scala | 2 +- 5 files changed, 20 insertions(+), 55 deletions(-) rename webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/{VolumeTracingDownsampling.scala => VolumeTracingMags.scala} (98%) diff --git a/app/controllers/AnnotationIOController.scala b/app/controllers/AnnotationIOController.scala index cf5bbaaea0f..42a17b90a74 100755 --- a/app/controllers/AnnotationIOController.scala +++ b/app/controllers/AnnotationIOController.scala @@ -32,7 +32,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeDataZipFo import com.scalableminds.webknossos.tracingstore.tracings.volume.{ VolumeDataZipFormat, VolumeTracingDefaults, - VolumeTracingDownsampling + VolumeTracingMags } import com.typesafe.scalalogging.LazyLogging @@ -336,7 +336,7 @@ class AnnotationIOController @Inject()( fallbackLayer = fallbackLayerOpt.map(_.name), largestSegmentId = combineLargestSegmentIdsByPrecedence(volumeTracing.largestSegmentId, fallbackLayerOpt.map(_.largestSegmentId)), - mags = VolumeTracingDownsampling.magsForVolumeTracing(dataSource, fallbackLayerOpt).map(vec3IntToProto), + mags = VolumeTracingMags.magsForVolumeTracing(dataSource, fallbackLayerOpt).map(vec3IntToProto), hasSegmentIndex = Some(tracingCanHaveSegmentIndex) ) } diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala index 9f7a48c0aed..8cee2a6f915 100755 --- a/app/models/annotation/AnnotationService.scala +++ b/app/models/annotation/AnnotationService.scala @@ -32,7 +32,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.VolumeDataZipFo import com.scalableminds.webknossos.tracingstore.tracings.volume.{ MagRestrictions, VolumeTracingDefaults, - VolumeTracingDownsampling + VolumeTracingMags } import com.typesafe.scalalogging.LazyLogging import models.annotation.AnnotationState._ @@ -72,7 +72,6 @@ class AnnotationService @Inject()( annotationInformationProvider: AnnotationInformationProvider, savedTracingInformationHandler: SavedTracingInformationHandler, annotationDAO: AnnotationDAO, - annotationLayersDAO: AnnotationLayerDAO, userDAO: UserDAO, taskTypeDAO: TaskTypeDAO, taskService: TaskService, @@ -131,7 +130,7 @@ class AnnotationService @Inject()( magRestrictions: MagRestrictions, mappingName: Option[String] ): Fox[VolumeTracing] = { - val mags = VolumeTracingDownsampling.magsForVolumeTracing(dataSource, fallbackLayer) + val mags = VolumeTracingMags.magsForVolumeTracing(dataSource, fallbackLayer) val magsRestricted = magRestrictions.filterAllowed(mags) val additionalAxes = fallbackLayer.map(_.additionalAxes).getOrElse(dataSource.additionalAxesUnion) @@ -306,10 +305,6 @@ class AnnotationService @Inject()( _ <- annotationDAO.insertOne(annotation) } yield annotation - def downsampleAnnotation(annotation: Annotation, volumeAnnotationLayer: AnnotationLayer)( - implicit ctx: DBAccessContext): Fox[Unit] = - ??? // TODO: remove feature or implement as update action - // WARNING: needs to be repeatable, might be called multiple times for an annotation def finish(annotation: Annotation, user: User, restrictions: AnnotationRestrictions)( implicit ctx: DBAccessContext): Fox[String] = { @@ -349,11 +344,6 @@ class AnnotationService @Inject()( }).flatten } - private def baseForTask(taskId: ObjectId)(implicit ctx: DBAccessContext): Fox[Annotation] = - (for { - list <- annotationDAO.findAllByTaskIdAndType(taskId, AnnotationType.TracingBase) - } yield list.headOption.toFox).flatten - def annotationsFor(taskId: ObjectId)(implicit ctx: DBAccessContext): Fox[List[Annotation]] = annotationDAO.findAllByTaskIdAndType(taskId, AnnotationType.Task) @@ -709,45 +699,13 @@ class AnnotationService @Inject()( updated <- annotationInformationProvider.provideAnnotation(typ, id, issuingUser) } yield updated - def resetToBase(annotation: Annotation)(implicit ctx: DBAccessContext, m: MessagesProvider): Fox[Unit] = // TODO: implement as update action? - annotation.typ match { - case AnnotationType.Explorational => - Fox.failure("annotation.revert.tasksOnly") - case AnnotationType.Task => - for { - task <- taskFor(annotation) - oldSkeletonTracingIdOpt <- annotation.skeletonTracingId // This also asserts that the annotation does not have multiple volume/skeleton layers - oldVolumeTracingIdOpt <- annotation.volumeTracingId - _ = logger.warn( - s"Resetting annotation ${annotation._id} to base, discarding skeleton tracing $oldSkeletonTracingIdOpt and/or volume tracing $oldVolumeTracingIdOpt") - annotationBase <- baseForTask(task._id) - dataset <- datasetDAO.findOne(annotationBase._dataset)(GlobalAccessContext) ?~> "dataset.notFoundForAnnotation" - (newSkeletonIdOpt, newVolumeIdOpt) <- tracingsFromBase(annotationBase, dataset) - _ <- Fox.bool2Fox(newSkeletonIdOpt.isDefined || newVolumeIdOpt.isDefined) ?~> "annotation.needsEitherSkeletonOrVolume" - _ <- Fox.runOptional(newSkeletonIdOpt)(newSkeletonId => - oldSkeletonTracingIdOpt.toFox.map { oldSkeletonId => - annotationLayersDAO.replaceTracingId(annotation._id, oldSkeletonId, newSkeletonId) - }) - _ <- Fox.runOptional(newVolumeIdOpt)(newVolumeId => - oldVolumeTracingIdOpt.toFox.map { oldVolumeId => - annotationLayersDAO.replaceTracingId(annotation._id, oldVolumeId, newVolumeId) - }) - } yield () - } - - private def tracingsFromBase(annotationBase: Annotation, dataset: Dataset)( - implicit ctx: DBAccessContext, - m: MessagesProvider): Fox[(Option[String], Option[String])] = + def resetToBase(annotation: Annotation)(implicit ctx: DBAccessContext): Fox[Unit] = for { - _ <- bool2Fox(dataset.isUsable) ?~> Messages("dataset.notImported", dataset.name) + _ <- bool2Fox(annotation.typ == AnnotationType.Task) ?~> "annotation.revert.tasksOnly" + dataset <- datasetDAO.findOne(annotation._dataset) tracingStoreClient <- tracingStoreService.clientFor(dataset) - baseSkeletonIdOpt <- annotationBase.skeletonTracingId - baseVolumeIdOpt <- annotationBase.volumeTracingId - newSkeletonId: Option[String] <- Fox.runOptional(baseSkeletonIdOpt)(skeletonId => - tracingStoreClient.duplicateSkeletonTracing(skeletonId)) - newVolumeId: Option[String] <- Fox.runOptional(baseVolumeIdOpt)(volumeId => - tracingStoreClient.duplicateVolumeTracing(volumeId)) - } yield (newSkeletonId, newVolumeId) + _ <- tracingStoreClient.resetToBase(annotation._id) ?~> "annotation.revert.failed" + } yield () private def settingsFor(annotation: Annotation)(implicit ctx: DBAccessContext) = if (annotation.typ == AnnotationType.Task || annotation.typ == AnnotationType.TracingBase) diff --git a/app/models/annotation/WKRemoteTracingStoreClient.scala b/app/models/annotation/WKRemoteTracingStoreClient.scala index b832300245f..58a5a9f8bff 100644 --- a/app/models/annotation/WKRemoteTracingStoreClient.scala +++ b/app/models/annotation/WKRemoteTracingStoreClient.scala @@ -241,4 +241,11 @@ class WKRemoteTracingStoreClient( } yield data } + def resetToBase(annotationId: ObjectId): Fox[Unit] = + for { + _ <- rpc(s"${tracingStore.url}/tracings/annotation/$annotationId/resetToBase").withLongTimeout + .addQueryString("token" -> RpcTokenHolder.webknossosToken) + .post() + } yield () + } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingMags.scala similarity index 98% rename from webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala rename to webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingMags.scala index 0210da35f84..c616069a119 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingDownsampling.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingMags.scala @@ -24,7 +24,7 @@ import scala.collection.mutable import scala.concurrent.ExecutionContext import scala.reflect.ClassTag -object VolumeTracingDownsampling { +object VolumeTracingMags { private def magsForVolumeTracingByLayerName(dataSource: DataSourceLike, fallbackLayerName: Option[String]): List[Vec3Int] = { val fallbackLayer: Option[DataLayerLike] = @@ -48,7 +48,7 @@ object VolumeTracingDownsampling { } } -trait VolumeTracingDownsampling +trait VolumeTracingMags extends BucketKeys with ProtoGeometryImplicits with VolumeBucketCompression @@ -270,7 +270,7 @@ trait VolumeTracingDownsampling implicit tc: TokenContext): Fox[List[Vec3Int]] = for { dataSource: DataSourceLike <- tracingStoreWkRpcClient.getDataSourceForTracing(oldTracingId) - magsForTracing = VolumeTracingDownsampling.magsForVolumeTracingByLayerName(dataSource, tracing.fallbackLayer) + magsForTracing = VolumeTracingMags.magsForVolumeTracingByLayerName(dataSource, tracing.fallbackLayer) } yield magsForTracing.sortBy(_.maxDim) protected def restrictMagList(tracing: VolumeTracing, magRestrictions: MagRestrictions): VolumeTracing = { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index bd8812f36a5..6ecf396ec85 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -61,7 +61,7 @@ class VolumeTracingService @Inject()( volumeSegmentIndexService: VolumeSegmentIndexService ) extends TracingService[VolumeTracing] with VolumeTracingBucketHelper - with VolumeTracingDownsampling + with VolumeTracingMags with WKWDataFormatHelper with FallbackDataHelper with DataFinder From bc51deb16b6282c4347e8dea7c168812fdee445b Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 5 Nov 2024 11:23:46 +0100 Subject: [PATCH 144/150] remove volume tracing downsampling feature (backend) --- .../tracings/volume/VolumeTracingMags.scala | 257 +----------------- .../volume/VolumeTracingService.scala | 24 +- 2 files changed, 9 insertions(+), 272 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingMags.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingMags.scala index c616069a119..1c7f316bb39 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingMags.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingMags.scala @@ -1,36 +1,13 @@ package com.scalableminds.webknossos.tracingstore.tracings.volume -import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.geometry.Vec3Int -import com.scalableminds.util.tools.{Fox, FoxImplicits} -import com.scalableminds.webknossos.datastore.models.{BucketPosition, UnsignedIntegerArray} -import com.scalableminds.webknossos.datastore.models.datasource.{DataLayerLike, DataSourceLike, ElementClass} -import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing.ElementClassProto -import com.scalableminds.webknossos.tracingstore.TSRemoteWebknossosClient +import com.scalableminds.webknossos.datastore.models.datasource.{DataLayerLike, DataSourceLike} import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing -import com.scalableminds.webknossos.tracingstore.tracings.{ - FossilDBClient, - KeyValueStoreImplicits, - TracingDataStore, - VersionedKeyValuePair -} -import net.liftweb.common.Empty import com.scalableminds.webknossos.datastore.geometry.{Vec3IntProto => ProtoPoint3D} import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits -import net.liftweb.common.Box import play.api.libs.json.{Format, Json} -import scala.collection.mutable -import scala.concurrent.ExecutionContext -import scala.reflect.ClassTag - -object VolumeTracingMags { - private def magsForVolumeTracingByLayerName(dataSource: DataSourceLike, - fallbackLayerName: Option[String]): List[Vec3Int] = { - val fallbackLayer: Option[DataLayerLike] = - fallbackLayerName.flatMap(name => dataSource.dataLayers.find(_.name == name)) - magsForVolumeTracing(dataSource, fallbackLayer) - } +object VolumeTracingMags extends ProtoGeometryImplicits { def magsForVolumeTracing(dataSource: DataSourceLike, fallbackLayer: Option[DataLayerLike]): List[Vec3Int] = { val fallbackLayerMags = fallbackLayer.map(_.resolutions) @@ -46,241 +23,15 @@ object VolumeTracingMags { } }.sortBy(_.maxDim) } -} - -trait VolumeTracingMags - extends BucketKeys - with ProtoGeometryImplicits - with VolumeBucketCompression - with KeyValueStoreImplicits - with ReversionHelper - with FoxImplicits { - - val tracingDataStore: TracingDataStore - val tracingStoreWkRpcClient: TSRemoteWebknossosClient - protected def saveBucket(dataLayer: VolumeTracingLayer, - bucket: BucketPosition, - data: Array[Byte], - version: Long, - toTemporaryStore: Boolean = false): Fox[Unit] - - protected def updateSegmentIndex(segmentIndexBuffer: VolumeSegmentIndexBuffer, - bucketPosition: BucketPosition, - bucketBytes: Array[Byte], - previousBucketBytesBox: Box[Array[Byte]], - elementClass: ElementClassProto, - mappingName: Option[String], - editableMappingTracingId: Option[String]): Fox[Unit] - - protected def editableMappingTracingId(tracing: VolumeTracing, tracingId: String): Option[String] - - protected def selectMappingName(tracing: VolumeTracing): Fox[Option[String]] - - protected def volumeSegmentIndexClient: FossilDBClient - - protected def downsampleWithLayer( - annotationId: String, // TODO required? - tracingId: String, - oldTracingId: String, - newTracing: VolumeTracing, - dataLayer: VolumeTracingLayer, - tracingService: VolumeTracingService)(implicit ec: ExecutionContext, tc: TokenContext): Fox[List[Vec3Int]] = { - val bucketVolume = 32 * 32 * 32 - for { - _ <- bool2Fox(newTracing.version == 0L) ?~> "Tracing has already been edited." - _ <- bool2Fox(newTracing.mags.nonEmpty) ?~> "Cannot downsample tracing with no mag list" - sourceMag = getSourceMag(newTracing) - magsToCreate <- getMagsToCreate(newTracing, oldTracingId) - elementClass = elementClassFromProto(newTracing.elementClass) - bucketDataMapMutable = new mutable.HashMap[BucketPosition, Array[Byte]]().withDefault(_ => revertedValue) - _ = fillMapWithSourceBucketsInplace(bucketDataMapMutable, tracingId, dataLayer, sourceMag) - originalBucketPositions = bucketDataMapMutable.keys.toList - updatedBucketsMutable = new mutable.ListBuffer[BucketPosition]() - _ = magsToCreate.foldLeft(sourceMag) { (previousMag, requiredMag) => - downsampleMagFromMag(previousMag, - requiredMag, - originalBucketPositions, - bucketDataMapMutable, - updatedBucketsMutable, - bucketVolume, - elementClass, - dataLayer) - requiredMag - } - fallbackLayer <- tracingService.getFallbackLayer(oldTracingId, newTracing) // remote wk does not know the new id yet - segmentIndexBuffer = new VolumeSegmentIndexBuffer(tracingId, - volumeSegmentIndexClient, - newTracing.version, - tracingService.remoteDatastoreClient, - fallbackLayer, - dataLayer.additionalAxes, - tc) - _ <- Fox.serialCombined(updatedBucketsMutable.toList) { bucketPosition: BucketPosition => - for { - _ <- saveBucket(dataLayer, bucketPosition, bucketDataMapMutable(bucketPosition), newTracing.version) - mappingName <- selectMappingName(newTracing) - _ <- Fox.runIfOptionTrue(newTracing.hasSegmentIndex)( - updateSegmentIndex( - segmentIndexBuffer, - bucketPosition, - bucketDataMapMutable(bucketPosition), - Empty, - newTracing.elementClass, - mappingName, - editableMappingTracingId(newTracing, tracingId) - )) - } yield () - } - _ <- segmentIndexBuffer.flush() - _ = logger.debug(s"Downsampled mags $magsToCreate from $sourceMag for volume tracing $tracingId.") - } yield sourceMag :: magsToCreate - } - - private def fillMapWithSourceBucketsInplace(bucketDataMap: mutable.Map[BucketPosition, Array[Byte]], - tracingId: String, - dataLayer: VolumeTracingLayer, - sourceMag: Vec3Int): Unit = { - val data: List[VersionedKeyValuePair[Array[Byte]]] = - tracingDataStore.volumeData.getMultipleKeys(None, Some(tracingId)) - data.foreach { keyValuePair: VersionedKeyValuePair[Array[Byte]] => - val bucketPositionOpt = parseBucketKey(keyValuePair.key, dataLayer.additionalAxes).map(_._2) - bucketPositionOpt.foreach { bucketPosition => - if (bucketPosition.mag == sourceMag) { - bucketDataMap(bucketPosition) = decompressIfNeeded(keyValuePair.value, - expectedUncompressedBucketSizeFor(dataLayer), - s"bucket $bucketPosition during downsampling") - } - } - } - } - - private def downsampleMagFromMag(previousMag: Vec3Int, - requiredMag: Vec3Int, - originalBucketPositions: List[BucketPosition], - bucketDataMapMutable: mutable.Map[BucketPosition, Array[Byte]], - updatedBucketsMutable: mutable.ListBuffer[BucketPosition], - bucketVolume: Int, - elementClass: ElementClass.Value, - dataLayer: VolumeTracingLayer): Unit = { - val downScaleFactor = - Vec3Int(requiredMag.x / previousMag.x, requiredMag.y / previousMag.y, requiredMag.z / previousMag.z) - downsampledBucketPositions(originalBucketPositions, requiredMag).foreach { downsampledBucketPosition => - val sourceBuckets: Seq[BucketPosition] = - sourceBucketPositionsFor(downsampledBucketPosition, downScaleFactor, previousMag) - val sourceData: Seq[Array[Byte]] = sourceBuckets.map(bucketDataMapMutable(_)) - val downsampledData: Array[Byte] = - if (sourceData.forall(_.sameElements(revertedValue))) - revertedValue - else { - val sourceDataFilled = fillZeroedIfNeeded(sourceData, bucketVolume, dataLayer.bytesPerElement) - val sourceDataTyped = UnsignedIntegerArray.fromByteArray(sourceDataFilled.toArray.flatten, elementClass) - val dataDownscaledTyped = - downsampleData(sourceDataTyped.grouped(bucketVolume).toArray, downScaleFactor, bucketVolume) - UnsignedIntegerArray.toByteArray(dataDownscaledTyped, elementClass) - } - bucketDataMapMutable(downsampledBucketPosition) = downsampledData - updatedBucketsMutable += downsampledBucketPosition - } - } - - private def downsampledBucketPositions(originalBucketPositions: List[BucketPosition], - requiredMag: Vec3Int): Set[BucketPosition] = - originalBucketPositions.map { bucketPosition: BucketPosition => - BucketPosition( - (bucketPosition.voxelMag1X / requiredMag.x / 32) * requiredMag.x * 32, - (bucketPosition.voxelMag1Y / requiredMag.y / 32) * requiredMag.y * 32, - (bucketPosition.voxelMag1Z / requiredMag.z / 32) * requiredMag.z * 32, - requiredMag, - bucketPosition.additionalCoordinates - ) - }.toSet - - private def sourceBucketPositionsFor(bucketPosition: BucketPosition, - downScaleFactor: Vec3Int, - previousMag: Vec3Int): Seq[BucketPosition] = - for { - z <- 0 until downScaleFactor.z - y <- 0 until downScaleFactor.y - x <- 0 until downScaleFactor.x - } yield { - BucketPosition( - bucketPosition.voxelMag1X + x * bucketPosition.bucketLength * previousMag.x, - bucketPosition.voxelMag1Y + y * bucketPosition.bucketLength * previousMag.y, - bucketPosition.voxelMag1Z + z * bucketPosition.bucketLength * previousMag.z, - previousMag, - bucketPosition.additionalCoordinates - ) - } - - private def fillZeroedIfNeeded(sourceData: Seq[Array[Byte]], - bucketVolume: Int, - bytesPerElement: Int): Seq[Array[Byte]] = - // Reverted buckets and missing buckets are represented by a single zero-byte. - // For downsampling, those need to be replaced with the full bucket volume of zero-bytes. - sourceData.map { sourceBucketData => - if (isRevertedElement(sourceBucketData)) { - Array.fill[Byte](bucketVolume * bytesPerElement)(0) - } else sourceBucketData - } - - private def downsampleData[T: ClassTag](data: Array[Array[T]], - downScaleFactor: Vec3Int, - bucketVolume: Int): Array[T] = { - val result = new Array[T](bucketVolume) - for { - z <- 0 until 32 - y <- 0 until 32 - x <- 0 until 32 - } { - val voxelSourceData: IndexedSeq[T] = for { - z_offset <- 0 until downScaleFactor.z - y_offset <- 0 until downScaleFactor.y - x_offset <- 0 until downScaleFactor.x - } yield { - val sourceVoxelPosition = - Vec3Int(x * downScaleFactor.x + x_offset, y * downScaleFactor.y + y_offset, z * downScaleFactor.z + z_offset) - val sourceBucketPosition = - Vec3Int(sourceVoxelPosition.x / 32, sourceVoxelPosition.y / 32, sourceVoxelPosition.z / 32) - val sourceVoxelPositionInSourceBucket = - Vec3Int(sourceVoxelPosition.x % 32, sourceVoxelPosition.y % 32, sourceVoxelPosition.z % 32) - val sourceBucketIndex = sourceBucketPosition.x + sourceBucketPosition.y * downScaleFactor.y + sourceBucketPosition.z * downScaleFactor.y * downScaleFactor.z - val sourceVoxelIndex = sourceVoxelPositionInSourceBucket.x + sourceVoxelPositionInSourceBucket.y * 32 + sourceVoxelPositionInSourceBucket.z * 32 * 32 - data(sourceBucketIndex)(sourceVoxelIndex) - } - result(x + y * 32 + z * 32 * 32) = mode(voxelSourceData) - } - result - } - - private def mode[T](items: Seq[T]): T = - items.groupBy(i => i).view.mapValues(_.size).maxBy(_._2)._1 - - private def getSourceMag(tracing: VolumeTracing): Vec3Int = - tracing.mags.minBy(_.maxDim) - - private def getMagsToCreate(tracing: VolumeTracing, oldTracingId: String)( - implicit tc: TokenContext): Fox[List[Vec3Int]] = - for { - requiredMags <- getRequiredMags(tracing, oldTracingId) - sourceMag = getSourceMag(tracing) - magsToCreate = requiredMags.filter(_.maxDim > sourceMag.maxDim) - } yield magsToCreate - - private def getRequiredMags(tracing: VolumeTracing, oldTracingId: String)( - implicit tc: TokenContext): Fox[List[Vec3Int]] = - for { - dataSource: DataSourceLike <- tracingStoreWkRpcClient.getDataSourceForTracing(oldTracingId) - magsForTracing = VolumeTracingMags.magsForVolumeTracingByLayerName(dataSource, tracing.fallbackLayer) - } yield magsForTracing.sortBy(_.maxDim) - protected def restrictMagList(tracing: VolumeTracing, magRestrictions: MagRestrictions): VolumeTracing = { + def restrictMagList(tracing: VolumeTracing, magRestrictions: MagRestrictions): VolumeTracing = { val tracingMags = resolveLegacyMagList(tracing.mags) val allowedMags = magRestrictions.filterAllowed(tracingMags.map(vec3IntFromProto)) tracing.withMags(allowedMags.map(vec3IntToProto)) } - protected def resolveLegacyMagList(mags: Seq[ProtoPoint3D]): Seq[ProtoPoint3D] = + def resolveLegacyMagList(mags: Seq[ProtoPoint3D]): Seq[ProtoPoint3D] = if (mags.isEmpty) Seq(ProtoPoint3D(1, 1, 1)) else mags } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 6ecf396ec85..3aea6db57be 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -61,7 +61,6 @@ class VolumeTracingService @Inject()( volumeSegmentIndexService: VolumeSegmentIndexService ) extends TracingService[VolumeTracing] with VolumeTracingBucketHelper - with VolumeTracingMags with WKWDataFormatHelper with FallbackDataHelper with DataFinder @@ -90,7 +89,7 @@ class VolumeTracingService @Inject()( private val fallbackLayerCache: AlfuCache[(String, Option[String], Option[String]), Option[RemoteFallbackLayer]] = AlfuCache(maxCapacity = 100) - override protected def updateSegmentIndex( + private def updateSegmentIndex( segmentIndexBuffer: VolumeSegmentIndexBuffer, bucketPosition: BucketPosition, bucketBytes: Array[Byte], @@ -173,10 +172,10 @@ class VolumeTracingService @Inject()( } } yield volumeTracing - override def editableMappingTracingId(tracing: VolumeTracing, tracingId: String): Option[String] = + def editableMappingTracingId(tracing: VolumeTracing, tracingId: String): Option[String] = if (tracing.getHasEditableMapping) Some(tracingId) else None - def selectMappingName(tracing: VolumeTracing): Fox[Option[String]] = + private def selectMappingName(tracing: VolumeTracing): Fox[Option[String]] = if (tracing.getHasEditableMapping) Fox.failure("mappingName called on volumeTracing with editableMapping!") else Fox.successful(tracing.mappingName) @@ -495,7 +494,7 @@ class VolumeTracingService @Inject()( editRotation: Option[Vec3Double], newVersion: Long)(implicit ec: ExecutionContext, tc: TokenContext): Fox[VolumeTracing] = { val tracingWithBB = addBoundingBoxFromTaskIfRequired(sourceTracing, isFromTask, datasetBoundingBox) - val tracingWithMagRestrictions = restrictMagList(tracingWithBB, magRestrictions) + val tracingWithMagRestrictions = VolumeTracingMags.restrictMagList(tracingWithBB, magRestrictions) for { fallbackLayer <- getFallbackLayer(sourceTracingId, sourceTracing) hasSegmentIndex <- VolumeSegmentIndexService.canHaveSegmentIndex(remoteDatastoreClient, fallbackLayer) @@ -607,19 +606,6 @@ class VolumeTracingService @Inject()( toTemporaryStore) } yield id - // TODO use or remove - def downsample(annotationId: String, tracingId: String, oldTracingId: String, newTracing: VolumeTracing)( - implicit tc: TokenContext): Fox[Unit] = - for { - resultingMags <- downsampleWithLayer(annotationId, - tracingId, - oldTracingId, - newTracing, - volumeTracingLayer(tracingId, newTracing), - this) - _ <- updateMagList(tracingId, newTracing, resultingMags.toSet) - } yield () - def volumeBucketsAreEmpty(tracingId: String): Boolean = volumeDataStore.getMultipleKeys(None, Some(tracingId), limit = Some(1))(toBox).isEmpty @@ -829,7 +815,7 @@ class VolumeTracingService @Inject()( else { val magSet = magSetFromZipfile(zipFile) val magsDoMatch = - magSet.isEmpty || magSet == resolveLegacyMagList(tracing.mags).map(vec3IntFromProto).toSet + magSet.isEmpty || magSet == VolumeTracingMags.resolveLegacyMagList(tracing.mags).map(vec3IntFromProto).toSet if (!magsDoMatch) Fox.failure("annotation.volume.magssDoNotMatch") From 7150c2562640295304e0ddc4eba114d2ab3d81cd Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 5 Nov 2024 11:56:00 +0100 Subject: [PATCH 145/150] also duplicate annotation v0 --- .../annotation/TSAnnotationService.scala | 16 ++++++++++++++-- .../tracings/volume/VolumeTracingService.scala | 2 +- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 2b020c5abdc..ed50ed09882 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -575,7 +575,6 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } } - // TODO duplicate v0 as well? (if current version is not v0) def duplicate( annotationId: String, newAnnotationId: String, @@ -583,13 +582,26 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss isFromTask: Boolean, datasetBoundingBox: Option[BoundingBox])(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationProto] = for { + // Duplicate v0 + v0Annotation <- get(annotationId, Some(0L)) + v0NewLayers <- Fox.serialCombined(v0Annotation.annotationLayers)(layer => + duplicateLayer(annotationId, layer, v0Annotation.version, isFromTask, datasetBoundingBox)) + v0DuplicatedAnnotation = v0Annotation.copy(annotationLayers = v0NewLayers, + earliestAccessibleVersion = v0Annotation.version) + + _ <- tracingDataStore.annotations.put(newAnnotationId, v0Annotation.version, v0DuplicatedAnnotation) + + // Duplicate current currentAnnotation <- get(annotationId, version) newLayers <- Fox.serialCombined(currentAnnotation.annotationLayers)(layer => duplicateLayer(annotationId, layer, currentAnnotation.version, isFromTask, datasetBoundingBox)) - _ <- duplicateUpdates(annotationId, newAnnotationId) duplicatedAnnotation = currentAnnotation.copy(annotationLayers = newLayers, earliestAccessibleVersion = currentAnnotation.version) _ <- tracingDataStore.annotations.put(newAnnotationId, currentAnnotation.version, duplicatedAnnotation) + + // Duplicate updates + _ <- duplicateUpdates(annotationId, newAnnotationId) + } yield duplicatedAnnotation private def duplicateUpdates(annotationId: String, newAnnotationId: String)( diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 3aea6db57be..3ee4de5b733 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -873,7 +873,7 @@ class VolumeTracingService @Inject()( persist: Boolean): Fox[Unit] = if (tracingsWithIds.forall(tracingWithId => tracingWithId._1.getHasEditableMapping)) { for { - _ <- bool2Fox(persist) ?~> "Cannot merge editable mappings without “persist” (used by compound annotations)" + _ <- bool2Fox(persist) ?~> "Cannot merge editable mappings without “persist” (trying to merge compound annotations?)" remoteFallbackLayers <- Fox.serialCombined(tracingsWithIds)(tracingWithId => remoteFallbackLayerFromVolumeTracing(tracingWithId._1, tracingWithId._2)) remoteFallbackLayer <- remoteFallbackLayers.headOption.toFox From 3ffdf3b97df1a61cdd84c9a05c95e66050ff066e Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 5 Nov 2024 12:13:13 +0100 Subject: [PATCH 146/150] use SequenceUtils --- .../controllers/TSAnnotationController.scala | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index 2f543b04548..3ca55233653 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -1,5 +1,6 @@ package com.scalableminds.webknossos.tracingstore.controllers +import collections.SequenceUtils import com.google.inject.Inject import com.scalableminds.util.geometry.BoundingBox import com.scalableminds.util.tools.Fox @@ -167,9 +168,12 @@ class TSAnnotationController @Inject()( volumeLayers = annotations.flatMap(_.annotationLayers.filter(_.`type` == AnnotationLayerTypeProto.Volume)) newSkeletonId = TracingId.generate newVolumeId = TracingId.generate - mergedSkeletonName = allEqual(skeletonLayers.map(_.name)) + mergedSkeletonName = SequenceUtils + .findUniqueElement(skeletonLayers.map(_.name)) .getOrElse(AnnotationLayer.defaultSkeletonLayerName) - mergedVolumeName = allEqual(volumeLayers.map(_.name)).getOrElse(AnnotationLayer.defaultVolumeLayerName) + mergedVolumeName = SequenceUtils + .findUniqueElement(volumeLayers.map(_.name)) + .getOrElse(AnnotationLayer.defaultVolumeLayerName) // TODO: Merge updates? if so, iron out reverts? // TODO: Merge editable mappings volumeTracings <- annotationService @@ -222,10 +226,4 @@ class TSAnnotationController @Inject()( } } - // TODO generalize, mix with assertAllOnSame* - private def allEqual(str: Seq[String]): Option[String] = - // returns the str if all names are equal, None otherwise - str.headOption.map(name => str.forall(_ == name)).flatMap { _ => - str.headOption - } } From fb1b595b323439358fd80a421ca7ea911173584f Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 5 Nov 2024 12:17:52 +0100 Subject: [PATCH 147/150] cleanup todo comments --- app/models/annotation/WKRemoteTracingStoreClient.scala | 4 +--- .../datastore/services/DSRemoteTracingstoreClient.scala | 2 -- .../tracingstore/annotation/TSAnnotationService.scala | 4 ++-- .../tracingstore/controllers/TSAnnotationController.scala | 4 ++-- .../tracings/editablemapping/EditableMappingStreams.scala | 4 ++-- .../tracingstore/tracings/volume/VolumeTracingService.scala | 3 +-- 6 files changed, 8 insertions(+), 13 deletions(-) diff --git a/app/models/annotation/WKRemoteTracingStoreClient.scala b/app/models/annotation/WKRemoteTracingStoreClient.scala index 58a5a9f8bff..eb516b9ed5d 100644 --- a/app/models/annotation/WKRemoteTracingStoreClient.scala +++ b/app/models/annotation/WKRemoteTracingStoreClient.scala @@ -108,8 +108,7 @@ class WKRemoteTracingStoreClient( } // Used in task creation. History is dropped, new version will be zero. - // TODO: currently also used in resetToBase. Fix that. - def duplicateSkeletonTracing(skeletonTracingId: String, // TODO: might also need annotation id + def duplicateSkeletonTracing(skeletonTracingId: String, editPosition: Option[Vec3Int] = None, editRotation: Option[Vec3Double] = None, boundingBox: Option[BoundingBox] = None): Fox[String] = @@ -121,7 +120,6 @@ class WKRemoteTracingStoreClient( .postWithJsonResponse[String]() // Used in task creation. History is dropped, new version will be zero. - // TODO: currently also used in resetToBase. Fix that. def duplicateVolumeTracing(volumeTracingId: String, magRestrictions: MagRestrictions = MagRestrictions.empty, editPosition: Option[Vec3Int] = None, diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala index 42473d29a21..5bd69d4d7c9 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteTracingstoreClient.scala @@ -70,11 +70,9 @@ class DSRemoteTracingstoreClient @Inject()( rpc(s"$tracingStoreUri/tracings/volume/${getZarrVersionDependantSubPath(zarrVersion)}/json/$tracingId").withTokenFromContext .getWithJsonResponse[List[String]] - // TODO annotation id def getZGroup(tracingId: String, tracingStoreUri: String)(implicit tc: TokenContext): Fox[JsObject] = rpc(s"$tracingStoreUri/tracings/volume/zarr/$tracingId/.zgroup").withTokenFromContext.getWithJsonResponse[JsObject] - // TODO annotation id def getEditableMappingSegmentIdsForAgglomerate(tracingStoreUri: String, tracingId: String, agglomerateId: Long)( implicit tc: TokenContext): Fox[EditableMappingSegmentListResult] = rpc(s"$tracingStoreUri/tracings/mapping/$tracingId/segmentsForAgglomerate") diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index ed50ed09882..82c44609e27 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -157,7 +157,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss case a: EditableMappingUpdateAction => annotationWithTracings.applyEditableMappingAction(a) case a: RevertToVersionAnnotationAction => - revertToVersion(annotationId, annotationWithTracings, a, targetVersion) // TODO: if the revert action is not isolated, we need not the target version of all but the target version of this update + revertToVersion(annotationId, annotationWithTracings, a, targetVersion) // TODO double check that the revert action is isolated and targetVersion is that of group case _: ResetToBaseAnnotationAction => resetToBase(annotationId, annotationWithTracings, targetVersion) case _: BucketMutatingVolumeUpdateAction => @@ -269,7 +269,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss annotationId, annotationWithTracings, annotation.version, - targetVersion) // TODO: targetVersion must be set per update group, as reverts may come between these + targetVersion) // TODO double-check that targetVersion is set per update group, as reverts may come between these updated <- applyUpdatesGrouped(annotationWithTracingsAndMappings, annotationId, updatesGroupsRegrouped, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index 3ca55233653..0a8fb5081fe 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -174,8 +174,8 @@ class TSAnnotationController @Inject()( mergedVolumeName = SequenceUtils .findUniqueElement(volumeLayers.map(_.name)) .getOrElse(AnnotationLayer.defaultVolumeLayerName) - // TODO: Merge updates? if so, iron out reverts? - // TODO: Merge editable mappings + // TODO Merge updates? if so, iron out reverts? + // TODO Merge editable mappings volumeTracings <- annotationService .findMultipleVolumes(volumeLayers.map { l => Some(TracingSelector(l.tracingId)) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingStreams.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingStreams.scala index cae82ad526c..e9c7422ecd3 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingStreams.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingStreams.scala @@ -61,7 +61,7 @@ class VersionedAgglomerateToGraphIterator(prefix: String, case None => getNextNonRevertedGraph.get } nextGraph = None - // TODO: parse graph key? (=agglomerate id) + // TODO parse graph key? (=agglomerate id) (nextRes.key, nextRes.value, nextRes.version) } @@ -116,7 +116,7 @@ class VersionedSegmentToAgglomerateChunkIterator(prefix: String, case None => getNextNonRevertedChunk.get } nextChunk = None - // TODO: parse chunk key? + // TODO parse chunk key? (nextRes.key, nextRes.value, nextRes.version) } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala index 3ee4de5b733..02902f222ce 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingService.scala @@ -46,7 +46,6 @@ import scala.concurrent.duration._ class VolumeTracingService @Inject()( val tracingDataStore: TracingDataStore, - val tracingStoreWkRpcClient: TSRemoteWebknossosClient, val adHocMeshServiceHolder: AdHocMeshServiceHolder, implicit val temporaryTracingStore: TemporaryTracingStore[VolumeTracing], implicit val temporaryVolumeDataStore: TemporaryVolumeDataStore, @@ -878,7 +877,7 @@ class VolumeTracingService @Inject()( remoteFallbackLayerFromVolumeTracing(tracingWithId._1, tracingWithId._2)) remoteFallbackLayer <- remoteFallbackLayers.headOption.toFox _ <- bool2Fox(remoteFallbackLayers.forall(_ == remoteFallbackLayer)) ?~> "Cannot merge editable mappings based on different dataset layers" - // TODO: _ <- editableMappingService.merge(newTracingId, tracingsWithIds.map(_._2), remoteFallbackLayer) + // TODO _ <- editableMappingService.merge(newTracingId, tracingsWithIds.map(_._2), remoteFallbackLayer) } yield () } else if (tracingsWithIds.forall(tracingWithId => !tracingWithId._1.getHasEditableMapping)) { Fox.empty From 89c4ec28103118b16937c93139cf1ef83e85dc6c Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 5 Nov 2024 13:28:09 +0100 Subject: [PATCH 148/150] remap action tracing ids in duplicate --- .../annotation/TSAnnotationService.scala | 76 +++++++++++++------ .../annotation/UpdateActions.scala | 1 + .../SkeletonTracingController.scala | 8 +- .../controllers/VolumeTracingController.scala | 34 ++------- .../EditableMappingUpdateActions.scala | 8 +- .../updating/SkeletonUpdateActions.scala | 66 ++++++++-------- .../tracings/volume/VolumeUpdateActions.scala | 28 +++++++ 7 files changed, 134 insertions(+), 87 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 82c44609e27..af980386d86 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -582,10 +582,14 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss isFromTask: Boolean, datasetBoundingBox: Option[BoundingBox])(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationProto] = for { - // Duplicate v0 v0Annotation <- get(annotationId, Some(0L)) + + // Duplicate updates + tracingIdMap <- duplicateUpdates(annotationId, newAnnotationId, v0Annotation.annotationLayers.map(_.tracingId)) + + // Duplicate v0 v0NewLayers <- Fox.serialCombined(v0Annotation.annotationLayers)(layer => - duplicateLayer(annotationId, layer, v0Annotation.version, isFromTask, datasetBoundingBox)) + duplicateLayer(annotationId, layer, tracingIdMap, v0Annotation.version, isFromTask, datasetBoundingBox)) v0DuplicatedAnnotation = v0Annotation.copy(annotationLayers = v0NewLayers, earliestAccessibleVersion = v0Annotation.version) @@ -594,40 +598,62 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss // Duplicate current currentAnnotation <- get(annotationId, version) newLayers <- Fox.serialCombined(currentAnnotation.annotationLayers)(layer => - duplicateLayer(annotationId, layer, currentAnnotation.version, isFromTask, datasetBoundingBox)) + duplicateLayer(annotationId, layer, tracingIdMap, currentAnnotation.version, isFromTask, datasetBoundingBox)) duplicatedAnnotation = currentAnnotation.copy(annotationLayers = newLayers, earliestAccessibleVersion = currentAnnotation.version) _ <- tracingDataStore.annotations.put(newAnnotationId, currentAnnotation.version, duplicatedAnnotation) - // Duplicate updates - _ <- duplicateUpdates(annotationId, newAnnotationId) - } yield duplicatedAnnotation - private def duplicateUpdates(annotationId: String, newAnnotationId: String)( - implicit ec: ExecutionContext): Fox[Unit] = + private def duplicateUpdates(annotationId: String, newAnnotationId: String, v0TracingIds: Seq[String])( + implicit ec: ExecutionContext): Fox[Map[String, String]] = { + val tracingIdMapMutable = scala.collection.mutable.Map[String, String]() + v0TracingIds.foreach { v0TracingId => + tracingIdMapMutable.put(v0TracingId, TracingId.generate) + } // TODO memory: batch + for { - updatesAsBytes: Seq[(Long, Array[Byte])] <- tracingDataStore.annotationUpdates - .getMultipleVersionsAsVersionValueTuple(annotationId) - _ <- Fox.serialCombined(updatesAsBytes) { - case (version, updateBytes) => - tracingDataStore.annotationUpdates.put(newAnnotationId, version, updateBytes) + updateLists: Seq[(Long, List[UpdateAction])] <- tracingDataStore.annotationUpdates + .getMultipleVersionsAsVersionValueTuple(annotationId)(fromJsonBytes[List[UpdateAction]]) + _ <- Fox.serialCombined(updateLists) { + case (version, updateList) => + for { + updateListAdapted <- Fox.serialCombined(updateList) { + case a: AddLayerAnnotationAction => + for { + actionTracingId <- a.tracingId ?~> "duplicating addLayer without tracingId" + _ = if (!tracingIdMapMutable.contains(actionTracingId)) { + a.tracingId.foreach(actionTracingId => tracingIdMapMutable.put(actionTracingId, TracingId.generate)) + } + mappedTracingId <- tracingIdMapMutable.get(actionTracingId) ?~> "duplicating action for unknown layer" + } yield a.copy(tracingId = Some(mappedTracingId)) + case a: LayerUpdateAction => + for { + mappedTracingId <- tracingIdMapMutable.get(a.actionTracingId) ?~> "duplicating action for unknown layer" + } yield a.withActionTracingId(mappedTracingId) + } + _ <- tracingDataStore.annotationUpdates.put(newAnnotationId, version, Json.toJson(updateListAdapted)) + } yield () } - } yield () + } yield tracingIdMapMutable.toMap + } private def duplicateLayer(annotationId: String, layer: AnnotationLayerProto, + tracingIdMap: Map[String, String], version: Long, isFromTask: Boolean, datasetBoundingBox: Option[BoundingBox])(implicit ec: ExecutionContext, tc: TokenContext): Fox[AnnotationLayerProto] = for { - newTracingId <- layer.`type` match { + newTracingId <- tracingIdMap.get(layer.tracingId) ?~> "duplicate unknown layer" + _ <- layer.`type` match { case AnnotationLayerTypeProto.Volume => duplicateVolumeTracing(annotationId, layer.tracingId, version, + newTracingId, version, isFromTask, None, @@ -636,7 +662,15 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss None, None) case AnnotationLayerTypeProto.Skeleton => - duplicateSkeletonTracing(annotationId, layer.tracingId, version, version, isFromTask, None, None, None) + duplicateSkeletonTracing(annotationId, + layer.tracingId, + version, + newTracingId, + version, + isFromTask, + None, + None, + None) case AnnotationLayerTypeProto.Unrecognized(num) => Fox.failure(f"unrecognized annotation layer type: $num") } } yield layer.copy(tracingId = newTracingId) @@ -645,14 +679,14 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss sourceAnnotationId: String, sourceTracingId: String, sourceVersion: Long, + newTracingId: String, newVersion: Long, isFromTask: Boolean, boundingBox: Option[BoundingBox], datasetBoundingBox: Option[BoundingBox], magRestrictions: MagRestrictions, editPosition: Option[Vec3Int], - editRotation: Option[Vec3Double])(implicit ec: ExecutionContext, tc: TokenContext): Fox[String] = { - val newTracingId = TracingId.generate + editRotation: Option[Vec3Double])(implicit ec: ExecutionContext, tc: TokenContext): Fox[String] = for { sourceTracing <- findVolume(sourceAnnotationId, sourceTracingId, Some(sourceVersion)) newTracing <- volumeTracingService.adaptVolumeForDuplicate(sourceTracingId, @@ -671,7 +705,6 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss _ <- Fox.runIf(newTracing.getHasEditableMapping)( duplicateEditableMapping(sourceAnnotationId, sourceTracingId, newTracingId, sourceVersion, newVersion)) } yield newTracingId - } private def duplicateEditableMapping(sourceAnnotationId: String, sourceTracingId: String, @@ -692,12 +725,12 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss sourceAnnotationId: String, sourceTracingId: String, sourceVersion: Long, + newTracingId: String, newVersion: Long, isFromTask: Boolean, editPosition: Option[Vec3Int], editRotation: Option[Vec3Double], - boundingBox: Option[BoundingBox])(implicit ec: ExecutionContext, tc: TokenContext): Fox[String] = { - val newTracingId = TracingId.generate + boundingBox: Option[BoundingBox])(implicit ec: ExecutionContext, tc: TokenContext): Fox[String] = for { skeleton <- findSkeleton(sourceAnnotationId, sourceTracingId, Some(sourceVersion)) adaptedSkeleton = skeletonTracingService.adaptSkeletonForDuplicate(skeleton, @@ -708,6 +741,5 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss newVersion) _ <- tracingDataStore.skeletons.put(newTracingId, newVersion, adaptedSkeleton) } yield newTracingId - } } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala index c81be809cb7..e10789e0210 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateActions.scala @@ -56,6 +56,7 @@ trait ApplyImmediatelyUpdateAction extends UpdateAction trait LayerUpdateAction extends UpdateAction { def actionTracingId: String + def withActionTracingId(newTracingId: String): LayerUpdateAction } object UpdateAction { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala index 0c3bfbc1b1d..431bc0ead7e 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/SkeletonTracingController.scala @@ -6,7 +6,7 @@ import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.SkeletonTracing.{SkeletonTracing, SkeletonTracingOpt, SkeletonTracings} import com.scalableminds.webknossos.datastore.services.UserAccessRequest import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService -import com.scalableminds.webknossos.tracingstore.tracings.TracingSelector +import com.scalableminds.webknossos.tracingstore.tracings.{TracingId, TracingSelector} import com.scalableminds.webknossos.tracingstore.tracings.skeleton._ import com.scalableminds.webknossos.tracingstore.{TSRemoteWebknossosClient, TracingStoreAccessTokenService} import play.api.i18n.Messages @@ -101,7 +101,10 @@ class SkeletonTracingController @Inject()(skeletonTracingService: SkeletonTracin for { mergedTracing <- Fox.box2Fox(skeletonTracingService.merge(tracings.flatten)) processedTracing = skeletonTracingService.remapTooLargeTreeIds(mergedTracing) - newId <- skeletonTracingService.save(processedTracing, None, processedTracing.version, toTemporaryStore = !persist) + newId <- skeletonTracingService.save(processedTracing, + None, + processedTracing.version, + toTemporaryStore = !persist) } yield Ok(Json.toJson(newId)) } } @@ -126,6 +129,7 @@ class SkeletonTracingController @Inject()(skeletonTracingService: SkeletonTracin annotationId, sourceTracingId = tracingId, sourceVersion = newestSourceVersion, + newTracingId = TracingId.generate, newVersion = 0, editPosition = editPositionParsed, editRotation = editRotationParsed, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala index c7444ae982d..b3683464bff 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala @@ -8,40 +8,17 @@ import com.scalableminds.webknossos.datastore.VolumeTracing.{VolumeTracing, Volu import com.scalableminds.webknossos.datastore.controllers.Controller import com.scalableminds.webknossos.datastore.geometry.ListOfVec3IntProto import com.scalableminds.util.tools.JsonHelper.{boxFormat, optionFormat} -import com.scalableminds.webknossos.datastore.helpers.{ - GetSegmentIndexParameters, - ProtoGeometryImplicits, - SegmentStatisticsParameters -} +import com.scalableminds.webknossos.datastore.helpers.{GetSegmentIndexParameters, ProtoGeometryImplicits, SegmentStatisticsParameters} import com.scalableminds.webknossos.datastore.models.datasource.{AdditionalAxis, DataLayer} -import com.scalableminds.webknossos.datastore.models.{ - LengthUnit, - VoxelSize, - WebknossosAdHocMeshRequest, - WebknossosDataRequest -} +import com.scalableminds.webknossos.datastore.models.{LengthUnit, VoxelSize, WebknossosAdHocMeshRequest, WebknossosDataRequest} import com.scalableminds.webknossos.datastore.rpc.RPC import com.scalableminds.webknossos.datastore.services.{FullMeshRequest, UserAccessRequest} import com.scalableminds.webknossos.tracingstore.annotation.{AnnotationTransactionService, TSAnnotationService} import com.scalableminds.webknossos.tracingstore.slacknotification.TSSlackNotificationService import com.scalableminds.webknossos.tracingstore.tracings.editablemapping.EditableMappingService -import com.scalableminds.webknossos.tracingstore.tracings.volume.{ - ImportVolumeDataVolumeAction, - MagRestrictions, - MergedVolumeStats, - TSFullMeshService, - VolumeDataZipFormat, - VolumeSegmentIndexService, - VolumeSegmentStatisticsService, - VolumeTracingService -} -import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingSelector} -import com.scalableminds.webknossos.tracingstore.{ - TSRemoteDatastoreClient, - TSRemoteWebknossosClient, - TracingStoreAccessTokenService, - TracingStoreConfig -} +import com.scalableminds.webknossos.tracingstore.tracings.volume.{ImportVolumeDataVolumeAction, MagRestrictions, MergedVolumeStats, TSFullMeshService, VolumeDataZipFormat, VolumeSegmentIndexService, VolumeSegmentStatisticsService, VolumeTracingService} +import com.scalableminds.webknossos.tracingstore.tracings.{KeyValueStoreImplicits, TracingId, TracingSelector} +import com.scalableminds.webknossos.tracingstore.{TSRemoteDatastoreClient, TSRemoteWebknossosClient, TracingStoreAccessTokenService, TracingStoreConfig} import net.liftweb.common.Empty import play.api.i18n.Messages import play.api.libs.Files.TemporaryFile @@ -400,6 +377,7 @@ class VolumeTracingController @Inject()( annotationId, sourceTracingId = tracingId, sourceVersion = newestSourceVersion, + newTracingId = TracingId.generate, newVersion = 0, editPosition = editPositionParsed, editRotation = editRotationParsed, diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala index 2bbbda3643a..9cd4ae51d32 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingUpdateActions.scala @@ -21,11 +21,11 @@ case class SplitAgglomerateUpdateAction(agglomerateId: Long, info: Option[String] = None) extends EditableMappingUpdateAction { override def addTimestamp(timestamp: Long): EditableMappingUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) } object SplitAgglomerateUpdateAction { @@ -47,11 +47,11 @@ case class MergeAgglomerateUpdateAction(agglomerateId1: Long, info: Option[String] = None) extends EditableMappingUpdateAction { override def addTimestamp(timestamp: Long): EditableMappingUpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) } object MergeAgglomerateUpdateAction { diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala index 4b7a79a4be8..1ca66e9f5cd 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/skeleton/updating/SkeletonUpdateActions.scala @@ -54,6 +54,8 @@ case class CreateTreeSkeletonAction(id: Int, override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) } case class DeleteTreeSkeletonAction(id: Int, @@ -67,11 +69,11 @@ case class DeleteTreeSkeletonAction(id: Int, override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) } case class UpdateTreeSkeletonAction(id: Int, @@ -110,6 +112,8 @@ case class UpdateTreeSkeletonAction(id: Int, override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) } case class MergeTreeSkeletonAction(sourceId: Int, @@ -137,11 +141,11 @@ case class MergeTreeSkeletonAction(sourceId: Int, override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) } case class MoveTreeComponentSkeletonAction(nodeIds: List[Int], @@ -179,11 +183,11 @@ case class MoveTreeComponentSkeletonAction(nodeIds: List[Int], override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) } case class CreateEdgeSkeletonAction(source: Int, @@ -202,11 +206,11 @@ case class CreateEdgeSkeletonAction(source: Int, override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) } case class DeleteEdgeSkeletonAction(source: Int, @@ -225,11 +229,12 @@ case class DeleteEdgeSkeletonAction(source: Int, override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) + } case class CreateNodeSkeletonAction(id: Int, @@ -272,11 +277,11 @@ case class CreateNodeSkeletonAction(id: Int, override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) } case class UpdateNodeSkeletonAction(id: Int, @@ -321,12 +326,11 @@ case class UpdateNodeSkeletonAction(id: Int, override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) - + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) } case class DeleteNodeSkeletonAction(nodeId: Int, @@ -347,11 +351,11 @@ case class DeleteNodeSkeletonAction(nodeId: Int, override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) } case class UpdateTreeGroupsSkeletonAction(treeGroups: List[UpdateActionTreeGroup], @@ -366,11 +370,11 @@ case class UpdateTreeGroupsSkeletonAction(treeGroups: List[UpdateActionTreeGroup override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) } case class UpdateTracingSkeletonAction(activeNode: Option[Int], @@ -397,11 +401,11 @@ case class UpdateTracingSkeletonAction(activeNode: Option[Int], override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) override def isViewOnlyChange: Boolean = true } @@ -422,11 +426,11 @@ case class UpdateTreeVisibilitySkeletonAction(treeId: Int, override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) override def isViewOnlyChange: Boolean = true } @@ -463,11 +467,11 @@ case class UpdateTreeGroupVisibilitySkeletonAction(treeGroupId: Option[Int], override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) } case class UpdateTreeEdgesVisibilitySkeletonAction(treeId: Int, @@ -487,11 +491,11 @@ case class UpdateTreeEdgesVisibilitySkeletonAction(treeId: Int, override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) } case class UpdateUserBoundingBoxesSkeletonAction(boundingBoxes: List[NamedBoundingBox], @@ -505,11 +509,11 @@ case class UpdateUserBoundingBoxesSkeletonAction(boundingBoxes: List[NamedBoundi override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) } case class UpdateUserBoundingBoxVisibilitySkeletonAction(boundingBoxId: Option[Int], @@ -533,11 +537,11 @@ case class UpdateUserBoundingBoxVisibilitySkeletonAction(boundingBoxId: Option[I override def addTimestamp(timestamp: Long): UpdateAction = this.copy(actionTimestamp = Some(timestamp)) - override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) - override def addAuthorId(authorId: Option[String]): UpdateAction = this.copy(actionAuthorId = authorId) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) override def isViewOnlyChange: Boolean = true } diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala index a3e7ff236b2..20cb63206ed 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeUpdateActions.scala @@ -48,6 +48,8 @@ case class UpdateBucketVolumeAction(position: Vec3Int, override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) def withoutBase64Data: UpdateBucketVolumeAction = this.copy(base64Data = None) @@ -70,6 +72,8 @@ case class UpdateTracingVolumeAction( override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) override def isViewOnlyChange: Boolean = true @@ -95,6 +99,8 @@ case class UpdateUserBoundingBoxesVolumeAction(boundingBoxes: List[NamedBounding override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) override def applyOn(tracing: VolumeTracing): VolumeTracing = tracing.withUserBoundingBoxes(boundingBoxes.map(_.toProto)) @@ -111,6 +117,8 @@ case class UpdateUserBoundingBoxVisibilityVolumeAction(boundingBoxId: Option[Int override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) override def applyOn(tracing: VolumeTracing): VolumeTracing = { @@ -137,6 +145,8 @@ case class RemoveFallbackLayerVolumeAction(actionTracingId: String, override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) override def applyOn(tracing: VolumeTracing): VolumeTracing = tracing.clearFallbackLayer @@ -152,6 +162,8 @@ case class ImportVolumeDataVolumeAction(actionTracingId: String, override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) override def applyOn(tracing: VolumeTracing): VolumeTracing = tracing.copy(largestSegmentId = largestSegmentId) @@ -166,6 +178,8 @@ case class AddSegmentIndexVolumeAction(actionTracingId: String, override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) override def applyOn(tracing: VolumeTracing): VolumeTracing = tracing.copy(hasSegmentIndex = Some(true)) @@ -192,6 +206,8 @@ case class CreateSegmentVolumeAction(id: Long, override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) override def applyOn(tracing: VolumeTracing): VolumeTracing = { val newSegment = @@ -230,6 +246,8 @@ case class UpdateSegmentVolumeAction(id: Long, override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) override def applyOn(tracing: VolumeTracing): VolumeTracing = { def segmentTransform(segment: Segment): Segment = @@ -258,6 +276,8 @@ case class DeleteSegmentVolumeAction(id: Long, override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) override def applyOn(tracing: VolumeTracing): VolumeTracing = tracing.withSegments(tracing.segments.filter(_.segmentId != id)) @@ -274,6 +294,8 @@ case class DeleteSegmentDataVolumeAction(id: Long, override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) } case class UpdateMappingNameVolumeAction(mappingName: Option[String], @@ -288,6 +310,8 @@ case class UpdateMappingNameVolumeAction(mappingName: Option[String], override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) override def applyOn(tracing: VolumeTracing): VolumeTracing = if (tracing.mappingIsLocked.getOrElse(false)) tracing // cannot change mapping name if it is locked @@ -311,6 +335,8 @@ case class UpdateSegmentGroupsVolumeAction(segmentGroups: List[UpdateActionSegme override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) } // TODO this now exists only for UpdateBucket. Make it a slimmed down version of that rather than generic? @@ -325,6 +351,8 @@ case class CompactVolumeUpdateAction(name: String, override def addAuthorId(authorId: Option[String]): VolumeUpdateAction = this.copy(actionAuthorId = authorId) override def addInfo(info: Option[String]): UpdateAction = this.copy(info = info) + override def withActionTracingId(newTracingId: String): LayerUpdateAction = + this.copy(actionTracingId = newTracingId) } object CompactVolumeUpdateAction { From b89a7f6a088e558d913244935e4deedf01cbe271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= Date: Tue, 5 Nov 2024 16:22:25 +0100 Subject: [PATCH 149/150] re-add accidentally deleted function --- frontend/javascripts/admin/admin_rest_api.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts index b20e7978b7e..faa53e48e81 100644 --- a/frontend/javascripts/admin/admin_rest_api.ts +++ b/frontend/javascripts/admin/admin_rest_api.ts @@ -913,6 +913,17 @@ export function getUpdateActionLog( }); } +export function getNewestVersionForTracing( + tracingStoreUrl: string, + annotationId: string, +): Promise { + return doWithToken((token) => + Request.receiveJSON( + `${tracingStoreUrl}/tracings/annotation/${annotationId}/newestVersion?token=${token}`, + ).then((obj) => obj.version), + ); +} + export async function getNewestVersionOfTracing( tracingStoreUrl: string, annotationId: string, From 5089c69c62593b3c578256fac8c51668c70d2a51 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 7 Nov 2024 14:21:06 +0100 Subject: [PATCH 150/150] update todo comments --- .../tracingstore/annotation/TSAnnotationService.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index af980386d86..26fa3731a30 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -157,7 +157,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss case a: EditableMappingUpdateAction => annotationWithTracings.applyEditableMappingAction(a) case a: RevertToVersionAnnotationAction => - revertToVersion(annotationId, annotationWithTracings, a, targetVersion) // TODO double check that the revert action is isolated and targetVersion is that of group + revertToVersion(annotationId, annotationWithTracings, a, targetVersion) case _: ResetToBaseAnnotationAction => resetToBase(annotationId, annotationWithTracings, targetVersion) case _: BucketMutatingVolumeUpdateAction => @@ -269,7 +269,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss annotationId, annotationWithTracings, annotation.version, - targetVersion) // TODO double-check that targetVersion is set per update group, as reverts may come between these + targetVersion) // Note: this targetVersion is overwritten for each update group, see annotation.withNewUpdaters updated <- applyUpdatesGrouped(annotationWithTracingsAndMappings, annotationId, updatesGroupsRegrouped, @@ -421,7 +421,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss } private def flushUpdatedTracings(annotationWithTracings: AnnotationWithTracings)(implicit ec: ExecutionContext) = - // TODO skip some flushes to save disk space (e.g. skeletons only nth version, or only if requested?) + // TODO skip some flushes to save disk space (for non-updated layers) for { _ <- Fox.serialCombined(annotationWithTracings.getVolumes) { case (volumeTracingId, volumeTracing) =>