WIP: Unified annotation versioning

app/controllers/AnnotationIOController.scala

@@ -79,6 +79,7 @@ class AnnotationIOController @Inject()(
     extends Controller
     with FoxImplicits
     with ProtoGeometryImplicits
+    with AnnotationLayerPrecedence
     with LazyLogging {
   implicit val actorSystem: ActorSystem = ActorSystem()
 
@@ -145,7 +146,8 @@ class AnnotationIOController @Inject()(
                                                        mergedSkeletonLayers ::: mergedVolumeLayers,
                                                        AnnotationType.Explorational,
                                                        name,
-                                                       description)
+                                                       description,
+                                                       ObjectId.generate)
             _ = analyticsService.track(UploadAnnotationEvent(request.identity, annotation))
           } yield
             JsonOk(
@@ -332,9 +334,8 @@ class AnnotationIOController @Inject()(
         boundingBox = bbox,
         elementClass = elementClass,
         fallbackLayer = fallbackLayerOpt.map(_.name),
-        largestSegmentId =
-          annotationService.combineLargestSegmentIdsByPrecedence(volumeTracing.largestSegmentId,
-                                                                 fallbackLayerOpt.map(_.largestSegmentId)),
+        largestSegmentId = combineLargestSegmentIdsByPrecedence(volumeTracing.largestSegmentId,
+                                                                fallbackLayerOpt.map(_.largestSegmentId)),
         mags = VolumeTracingDownsampling.magsForVolumeTracing(dataSource, fallbackLayerOpt).map(vec3IntToProto),
         hasSegmentIndex = Some(tracingCanHaveSegmentIndex)
       )

app/controllers/LegacyApiController.scala

@@ -5,6 +5,7 @@ import play.silhouette.api.actions.SecuredRequest
 import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.datastore.models.VoxelSize
 import com.scalableminds.webknossos.datastore.models.annotation.{AnnotationLayer, AnnotationLayerType}
+import com.scalableminds.webknossos.tracingstore.annotation.AnnotationLayerParameters
 import com.scalableminds.webknossos.tracingstore.tracings.volume.MagRestrictions
 import models.dataset.DatasetService
 import models.organization.OrganizationDAO
@@ -128,7 +129,7 @@ class LegacyApiController @Inject()(annotationController: AnnotationController,
     } yield adaptedResult
   }
 
-  def annotationInfoV4(typ: String, id: String, timestamp: Long): Action[AnyContent] = sil.SecuredAction.async {
+  def annotationInfoV4(typ: String, id: String, timestamp: Option[Long]): Action[AnyContent] = sil.SecuredAction.async {
     implicit request =>
       for {
         _ <- Fox.successful(logVersioned(request))

app/controllers/UserTokenController.scala

@@ -11,7 +11,7 @@ import com.scalableminds.webknossos.datastore.services.{
   UserAccessAnswer,
   UserAccessRequest
 }
-import com.scalableminds.webknossos.tracingstore.tracings.TracingIds
+import com.scalableminds.webknossos.tracingstore.tracings.TracingId
 
 import javax.inject.Inject
 import models.annotation._
@@ -39,7 +39,6 @@ object RpcTokenHolder {
 
 class UserTokenController @Inject()(datasetDAO: DatasetDAO,
                                     datasetService: DatasetService,
-                                    annotationDAO: AnnotationDAO,
                                     annotationPrivateLinkDAO: AnnotationPrivateLinkDAO,
                                     userService: UserService,
                                     organizationDAO: OrganizationDAO,
@@ -98,6 +97,8 @@ class UserTokenController @Inject()(datasetDAO: DatasetDAO,
             handleDataSourceAccess(accessRequest.resourceId, accessRequest.mode, userBox)(sharingTokenAccessCtx)
           case AccessResourceType.tracing =>
             handleTracingAccess(accessRequest.resourceId.name, accessRequest.mode, userBox, token)
+          case AccessResourceType.annotation =>
+            handleAnnotationAccess(accessRequest.resourceId.name, accessRequest.mode, userBox, token)
           case AccessResourceType.jobExport =>
             handleJobExportAccess(accessRequest.resourceId.name, accessRequest.mode, userBox)
           case _ =>
@@ -159,7 +160,19 @@ class UserTokenController @Inject()(datasetDAO: DatasetDAO,
   private def handleTracingAccess(tracingId: String,
                                   mode: AccessMode,
                                   userBox: Box[User],
-                                  token: Option[String]): Fox[UserAccessAnswer] = {
+                                  token: Option[String]): Fox[UserAccessAnswer] =
+    if (tracingId == TracingId.dummy)
+      Fox.successful(UserAccessAnswer(granted = true))
+    else
+      for {
+        annotation <- annotationInformationProvider.annotationForTracing(tracingId)(GlobalAccessContext) ?~> "annotation.notFound"
+        result <- handleAnnotationAccess(annotation._id.toString, mode, userBox, token)
+      } yield result
+
+  private def handleAnnotationAccess(annotationId: String,
+                                     mode: AccessMode,
+                                     userBox: Box[User],
+                                     token: Option[String]): Fox[UserAccessAnswer] = {
     // Access is explicitly checked by userBox, not by DBAccessContext, as there is no token sharing for annotations
     // Optionally, an accessToken can be provided which explicitly looks up the read right the private link table
 
@@ -170,16 +183,15 @@ class UserTokenController @Inject()(datasetDAO: DatasetDAO,
         case _                => Fox.successful(false)
       }
 
-    if (tracingId == TracingIds.dummyTracingId)
+    if (annotationId == ObjectId.dummyId.toString) {
       Fox.successful(UserAccessAnswer(granted = true))
-    else {
+    } else {
       for {
-        annotation <- annotationInformationProvider.annotationForTracing(tracingId)(GlobalAccessContext) ?~> "annotation.notFound"
+        annotation <- annotationInformationProvider.provideAnnotation(annotationId, userBox)(GlobalAccessContext) ?~> "annotation.notFound"
         annotationAccessByToken <- token
           .map(annotationPrivateLinkDAO.findOneByAccessToken)
           .getOrElse(Fox.empty)
           .futureBox
-
         allowedByToken = annotationAccessByToken.exists(annotation._id == _._annotation)
         restrictions <- annotationInformationProvider.restrictionsFor(
           AnnotationIdentifier(annotation.typ, annotation._id))(GlobalAccessContext) ?~> "restrictions.notFound"
@@ -201,7 +213,7 @@ class UserTokenController @Inject()(datasetDAO: DatasetDAO,
         jobBox <- jobDAO.findOne(jobIdValidated)(DBAccessContext(userBox)).futureBox
         answer = jobBox match {
           case Full(_) => UserAccessAnswer(granted = true)
-          case _       => UserAccessAnswer(granted = false, Some(s"No ${mode} access to job export"))
+          case _       => UserAccessAnswer(granted = false, Some(s"No $mode access to job export"))
         }
       } yield answer
     }

app/controllers/WKRemoteTracingStoreController.scala

@@ -3,17 +3,25 @@ package controllers
 import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext}
 import com.scalableminds.util.time.Instant
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
+import com.scalableminds.webknossos.datastore.Annotation.AnnotationProto
+import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing
+import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing
+import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayer
 import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId
 import com.scalableminds.webknossos.tracingstore.TracingUpdatesReport
+import com.scalableminds.webknossos.tracingstore.annotation.AnnotationLayerParameters
+import com.scalableminds.webknossos.tracingstore.tracings.TracingId
 
 import javax.inject.Inject
 import models.analytics.{AnalyticsService, UpdateAnnotationEvent, UpdateAnnotationViewOnlyEvent}
 import models.annotation.AnnotationState._
 import models.annotation.{
   Annotation,
   AnnotationDAO,
+  AnnotationDefaults,
   AnnotationInformationProvider,
   AnnotationLayerDAO,
+  AnnotationService,
   TracingDataSourceTemporaryStore,
   TracingStoreService
 }
@@ -24,8 +32,9 @@ import models.user.time.TimeSpanService
 import play.api.i18n.Messages
 import play.api.libs.json.Json
 import play.api.mvc.{Action, AnyContent, PlayBodyParsers}
+import scalapb.GeneratedMessage
 import security.{WebknossosBearerTokenAuthenticatorService, WkSilhouetteEnvironment}
-import utils.WkConf
+import utils.{ObjectId, WkConf}
 
 import scala.concurrent.ExecutionContext
 
@@ -37,6 +46,7 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore
                                                userDAO: UserDAO,
                                                annotationInformationProvider: AnnotationInformationProvider,
                                                analyticsService: AnalyticsService,
+                                               annotationService: AnnotationService,
                                                datasetDAO: DatasetDAO,
                                                annotationDAO: AnnotationDAO,
                                                annotationLayerDAO: AnnotationLayerDAO,
@@ -50,18 +60,47 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore
   val bearerTokenService: WebknossosBearerTokenAuthenticatorService =
     wkSilhouetteEnvironment.combinedAuthenticatorService.tokenAuthenticatorService
 
+  def updateAnnotation(name: String, key: String, annotationId: String): Action[AnnotationProto] =
+    Action.async(validateProto[AnnotationProto]) { implicit request =>
+      // tracingstore only sends this request after ensuring write access
+      implicit val ctx: DBAccessContext = GlobalAccessContext
+      for {
+        annotationIdValidated <- ObjectId.fromString(annotationId)
+        existingLayers <- annotationLayerDAO.findAnnotationLayersFor(annotationIdValidated)
+        newLayersProto = request.body.layers
+        existingLayerIds = existingLayers.map(_.tracingId).toSet
+        newLayerIds = newLayersProto.map(_.tracingId).toSet
+        layerIdsToDelete = existingLayerIds.diff(newLayerIds)
+        layerIdsToUpdate = existingLayerIds.intersect(newLayerIds)
+        layerIdsToInsert = newLayerIds.diff(existingLayerIds)
+        _ <- Fox.serialCombined(layerIdsToDelete.toList)(
+          annotationLayerDAO.deleteOneByTracingId(annotationIdValidated, _))
+        _ <- Fox.serialCombined(newLayersProto.filter(l => layerIdsToInsert.contains(l.tracingId))) { layerProto =>
+          annotationLayerDAO.insertOne(annotationIdValidated, AnnotationLayer.fromProto(layerProto))
+        }
+        _ <- Fox.serialCombined(newLayersProto.filter(l => layerIdsToUpdate.contains(l.tracingId)))(l =>
+          annotationLayerDAO.updateName(annotationIdValidated, l.tracingId, l.name))
+        // Layer stats are ignored here, they are sent eagerly when saving updates
+        _ <- annotationDAO.updateName(annotationIdValidated,
+                                      request.body.name.getOrElse(AnnotationDefaults.defaultName))
+        _ <- annotationDAO.updateDescription(annotationIdValidated,
+                                             request.body.description.getOrElse(AnnotationDefaults.defaultDescription))
+      } yield Ok
+    }
+
   def handleTracingUpdateReport(name: String, key: String): Action[TracingUpdatesReport] =
     Action.async(validateJson[TracingUpdatesReport]) { implicit request =>
       implicit val ctx: DBAccessContext = GlobalAccessContext
       tracingStoreService.validateAccess(name, key) { _ =>
         val report = request.body
         for {
-          annotation <- annotationDAO.findOneByTracingId(report.tracingId)
+          annotationId <- ObjectId.fromString(report.annotationId)
+          annotation <- annotationDAO.findOne(annotationId)
           _ <- ensureAnnotationNotFinished(annotation)
           _ <- annotationDAO.updateModified(annotation._id, Instant.now)
-          _ <- Fox.runOptional(report.statistics) { statistics =>
-            annotationLayerDAO.updateStatistics(annotation._id, report.tracingId, statistics)
-          }
+          /*_ <- Fox.runOptional(report.statistics) { statistics =>
+            annotationLayerDAO.updateStatistics(annotation._id, annotationId, statistics)
+          }*/ // TODO stats per tracing id. note: they might arrive before the layer is created. skip them then.
           userBox <- bearerTokenService.userForTokenOpt(report.userToken).futureBox
           trackTime = report.significantChangesCount > 0 || !wkConf.WebKnossos.User.timeTrackingOnlyWithSignificantChanges
           _ <- Fox.runOptional(userBox)(user =>
@@ -113,6 +152,20 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore
       }
     }
 
+  def annotationIdForTracing(name: String, key: String, tracingId: String): Action[AnyContent] =
+    Action.async { implicit request =>
+      tracingStoreService.validateAccess(name, key) { _ =>
+        implicit val ctx: DBAccessContext = GlobalAccessContext
+        if (tracingId == TracingId.dummy) {
+          Fox.successful(Ok(Json.toJson(ObjectId.dummyId)))
+        } else {
+          for {
+            annotation <- annotationInformationProvider.annotationForTracing(tracingId) ?~> s"No annotation for tracing $tracingId"
+          } yield Ok(Json.toJson(annotation._id))
+        }
+      }
+    }
+
   def dataStoreUriForDataset(name: String,
                              key: String,
                              organizationId: Option[String],
@@ -131,4 +184,28 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore
         } yield Ok(Json.toJson(dataStore.url))
       }
     }
+
+  def createTracing(name: String,
+                    key: String,
+                    annotationId: String,
+                    previousVersion: Long): Action[AnnotationLayerParameters] =
+    Action.async(validateJson[AnnotationLayerParameters]) { implicit request =>
+      tracingStoreService.validateAccess(name, key) { _ =>
+        implicit val ctx: DBAccessContext = GlobalAccessContext
+        for {
+          annotationIdValidated <- ObjectId.fromString(annotationId)
+          annotation <- annotationDAO.findOne(annotationIdValidated) ?~> "annotation.notFound"
+          dataset <- datasetDAO.findOne(annotation._dataset)
+          tracingEither <- annotationService.createTracingForExplorational(dataset,
+                                                                           request.body,
+                                                                           annotation.annotationLayers,
+                                                                           Some(previousVersion))
+          tracing: GeneratedMessage = tracingEither match {
+            case Left(s: SkeletonTracing) => s
+            case Right(v: VolumeTracing)  => v
+          }
+        } yield Ok(tracing.toByteArray).as(protobufMimeType)
+      }
+    }
+
 }

app/models/analytics/AnalyticsService.scala

@@ -55,6 +55,7 @@ class AnalyticsService @Inject()(rpc: RPC,
       }
       val wrappedJson = Json.obj("api_key" -> conf.key, "events" -> List(analyticsEventJson))
       rpc(conf.uri).silent.postJson(wrappedJson)
+      ()
     }
     Fox.successful(())
   }

app/models/annotation/Annotation.scala

@@ -22,16 +22,21 @@ import javax.inject.Inject
 import scala.concurrent.ExecutionContext
 import scala.concurrent.duration.FiniteDuration
 
+object AnnotationDefaults {
+  val defaultName: String = ""
+  val defaultDescription: String = ""
+}
+
 case class Annotation(
     _id: ObjectId,
     _dataset: ObjectId,
     _task: Option[ObjectId] = None,
     _team: ObjectId,
     _user: ObjectId,
     annotationLayers: List[AnnotationLayer],
-    description: String = "",
+    description: String = AnnotationDefaults.defaultDescription,
     visibility: AnnotationVisibility.Value = AnnotationVisibility.Internal,
-    name: String = "",
+    name: String = AnnotationDefaults.defaultName,
     viewConfiguration: Option[JsObject] = None,
     state: AnnotationState.Value = Active,
     isLockedByOwner: Boolean = false,
@@ -140,13 +145,20 @@ class AnnotationLayerDAO @Inject()(SQLClient: SqlClient)(implicit ec: ExecutionC
     q"""INSERT INTO webknossos.annotation_layers(_annotation, tracingId, typ, name, statistics)
           VALUES($annotationId, ${a.tracingId}, ${a.typ}, ${a.name}, ${a.stats})""".asUpdate
 
-  def deleteOne(annotationId: ObjectId, layerName: String): Fox[Unit] =
+  def deleteOneByName(annotationId: ObjectId, layerName: String): Fox[Unit] =
     for {
       _ <- run(q"""DELETE FROM webknossos.annotation_layers
                    WHERE _annotation = $annotationId
                    AND name = $layerName""".asUpdate)
     } yield ()
 
+  def deleteOneByTracingId(annotationId: ObjectId, tracingId: String): Fox[Unit] =
+    for {
+      _ <- run(q"""DELETE FROM webknossos.annotation_layers
+                   WHERE _annotation = $annotationId
+                   AND tracingId = $tracingId""".asUpdate)
+    } yield ()
+
   def findAnnotationIdByTracingId(tracingId: String): Fox[ObjectId] =
     for {
       rList <- run(q"SELECT _annotation FROM webknossos.annotation_layers WHERE tracingId = $tracingId".as[ObjectId])
@@ -503,6 +515,18 @@ class AnnotationDAO @Inject()(sqlClient: SqlClient, annotationLayerDAO: Annotati
                    AND a.typ = ${AnnotationType.Task} """.as[ObjectId])
     } yield r.toList
 
+  def findBaseIdForTask(taskId: ObjectId)(implicit ctx: DBAccessContext): Fox[ObjectId] =
+    for {
+      accessQuery <- readAccessQuery
+      r <- run(q"""SELECT _id
+                   FROM $existingCollectionName
+                   WHERE _task = $taskId
+                   AND typ = ${AnnotationType.TracingBase}
+                   AND state != ${AnnotationState.Cancelled}
+                   AND $accessQuery""".as[ObjectId])
+      firstRow <- r.headOption
+    } yield firstRow
+
   def findAllByTaskIdAndType(taskId: ObjectId, typ: AnnotationType)(
       implicit ctx: DBAccessContext): Fox[List[Annotation]] =
     for {