Merge RemoteSourceDescriptorService into DataVaultService

test/backend/DataVaultTestSuite.scala

@@ -14,7 +14,7 @@ import com.scalableminds.webknossos.datastore.datavault.{
   S3DataVault,
   VaultPath
 }
-import com.scalableminds.webknossos.datastore.storage.{GoogleServiceAccountCredential, RemoteSourceDescriptor}
+import com.scalableminds.webknossos.datastore.storage.{GoogleServiceAccountCredential, CredentializedUPath}
 import com.scalableminds.util.tools.{Box, Empty, EmptyBox, Failure, Full}
 import com.scalableminds.webknossos.datastore.helpers.UPath
 import play.api.libs.json.JsString
@@ -41,7 +41,7 @@ class DataVaultTestSuite extends PlaySpec {
           WsTestClient.withClient { ws =>
             val upath = UPath.fromStringUnsafe("http://storage.googleapis.com/")
             val vaultPath =
-              new VaultPath(upath, HttpsDataVault.create(RemoteSourceDescriptor(upath, None), ws, dummyDataStoreHost))
+              new VaultPath(upath, HttpsDataVault.create(CredentializedUPath(upath, None), ws, dummyDataStoreHost))
             val bytes =
               (vaultPath / s"neuroglancer-fafb-data/fafb_v14/fafb_v14_orig/$dataKey")
                 .readBytes(Some(range))(globalExecutionContext, emptyTokenContext)
@@ -55,7 +55,7 @@ class DataVaultTestSuite extends PlaySpec {
 
       "using Google Cloud Storage Vault" should {
         val upath = UPath.fromStringUnsafe("gs://neuroglancer-fafb-data/fafb_v14/fafb_v14_orig")
-        val vaultPath = new VaultPath(upath, GoogleCloudDataVault.create(RemoteSourceDescriptor(upath, None)))
+        val vaultPath = new VaultPath(upath, GoogleCloudDataVault.create(CredentializedUPath(upath, None)))
         "return correct response" in {
 
           val bytes = (vaultPath / dataKey)
@@ -87,7 +87,7 @@ class DataVaultTestSuite extends PlaySpec {
               new VaultPath(
                 upath,
                 GoogleCloudDataVault.create(
-                  RemoteSourceDescriptor(
+                  CredentializedUPath(
                     upath,
                     Some(GoogleServiceAccountCredential("name", JsString("secret"), Some("user"), Some("org")))))
               )
@@ -104,7 +104,7 @@ class DataVaultTestSuite extends PlaySpec {
           val upath = UPath.fromStringUnsafe("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5/em/fibsem-uint16/")
           WsTestClient.withClient { ws =>
             val vaultPath =
-              new VaultPath(upath, S3DataVault.create(RemoteSourceDescriptor(upath, None), ws)(globalExecutionContext))
+              new VaultPath(upath, S3DataVault.create(CredentializedUPath(upath, None), ws)(globalExecutionContext))
             val bytes =
               (vaultPath / "s0/5/5/5")
                 .readBytes(Some(range))(globalExecutionContext, emptyTokenContext)
@@ -125,7 +125,7 @@ class DataVaultTestSuite extends PlaySpec {
           WsTestClient.withClient { ws =>
             val upath = UPath.fromStringUnsafe("http://storage.googleapis.com/")
             val vaultPath =
-              new VaultPath(upath, HttpsDataVault.create(RemoteSourceDescriptor(upath, None), ws, dummyDataStoreHost))
+              new VaultPath(upath, HttpsDataVault.create(CredentializedUPath(upath, None), ws, dummyDataStoreHost))
             val bytes = (vaultPath / s"neuroglancer-fafb-data/fafb_v14/fafb_v14_orig/$dataKey")
               .readBytes()(globalExecutionContext, emptyTokenContext)
               .get(handleFoxJustification)
@@ -139,7 +139,7 @@ class DataVaultTestSuite extends PlaySpec {
       "using Google Cloud Storage Vault" should {
         "return correct response" in {
           val upath = UPath.fromStringUnsafe("gs://neuroglancer-fafb-data/fafb_v14/fafb_v14_orig")
-          val vaultPath = new VaultPath(upath, GoogleCloudDataVault.create(RemoteSourceDescriptor(upath, None)))
+          val vaultPath = new VaultPath(upath, GoogleCloudDataVault.create(CredentializedUPath(upath, None)))
           val bytes =
             (vaultPath / dataKey).readBytes()(globalExecutionContext, emptyTokenContext).get(handleFoxJustification)
 
@@ -153,7 +153,7 @@ class DataVaultTestSuite extends PlaySpec {
           val upath = UPath.fromStringUnsafe("s3://open-neurodata/bock11/image/4_4_40")
           WsTestClient.withClient { ws =>
             val vaultPath =
-              new VaultPath(upath, S3DataVault.create(RemoteSourceDescriptor(upath, None), ws)(globalExecutionContext))
+              new VaultPath(upath, S3DataVault.create(CredentializedUPath(upath, None), ws)(globalExecutionContext))
             val bytes =
               (vaultPath / "33792-34304_29696-30208_3216-3232")
                 .readBytes()(globalExecutionContext, emptyTokenContext)
@@ -166,7 +166,7 @@ class DataVaultTestSuite extends PlaySpec {
           "requesting a non-existent bucket" in {
             val upath = UPath.fromStringUnsafe(s"s3://non-existent-bucket${UUID.randomUUID}/non-existent-object")
             WsTestClient.withClient { ws =>
-              val s3DataVault = S3DataVault.create(RemoteSourceDescriptor(upath, None), ws)(globalExecutionContext)
+              val s3DataVault = S3DataVault.create(CredentializedUPath(upath, None), ws)(globalExecutionContext)
               val vaultPath = new VaultPath(upath, s3DataVault)
               val result =
                 vaultPath.readBytes()(globalExecutionContext, emptyTokenContext).await(handleFoxJustification)
@@ -179,7 +179,7 @@ class DataVaultTestSuite extends PlaySpec {
           "requesting a non-existent object in existent bucket" in {
             val upath = UPath.fromStringUnsafe(s"s3://open-neurodata/non-existent-object${UUID.randomUUID}")
             WsTestClient.withClient { ws =>
-              val s3DataVault = S3DataVault.create(RemoteSourceDescriptor(upath, None), ws)(globalExecutionContext)
+              val s3DataVault = S3DataVault.create(CredentializedUPath(upath, None), ws)(globalExecutionContext)
               val vaultPath = new VaultPath(upath, s3DataVault)
               val result =
                 vaultPath.readBytes()(globalExecutionContext, emptyTokenContext).await(handleFoxJustification)
@@ -194,7 +194,7 @@ class DataVaultTestSuite extends PlaySpec {
       val upath = UPath.fromStringUnsafe("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5/em/fibsem-uint16/")
       WsTestClient.withClient { ws =>
         val vaultPath =
-          new VaultPath(upath, S3DataVault.create(RemoteSourceDescriptor(upath, None), ws)(globalExecutionContext))
+          new VaultPath(upath, S3DataVault.create(CredentializedUPath(upath, None), ws)(globalExecutionContext))
 
         "using s3 data vault" should {
           "list available directories" in {
@@ -208,7 +208,7 @@ class DataVaultTestSuite extends PlaySpec {
           "return failure" when {
             "requesting directory listing on non-existent bucket" in {
               val upath = UPath.fromStringUnsafe(f"s3://non-existent-bucket${UUID.randomUUID}/non-existent-object/")
-              val s3DataVault = S3DataVault.create(RemoteSourceDescriptor(upath, None), ws)(globalExecutionContext)
+              val s3DataVault = S3DataVault.create(CredentializedUPath(upath, None), ws)(globalExecutionContext)
               val vaultPath = new VaultPath(upath, s3DataVault)
               val result = vaultPath.listDirectory(maxItems = 5)(globalExecutionContext).await(handleFoxJustification)
               assertBoxFailure(result)

webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreModule.scala

@@ -28,7 +28,7 @@ import com.scalableminds.webknossos.datastore.services.segmentindex.{
   ZarrSegmentIndexFileService
 }
 import com.scalableminds.webknossos.datastore.services.uploading.UploadService
-import com.scalableminds.webknossos.datastore.storage.{DataVaultService, RemoteSourceDescriptorService}
+import com.scalableminds.webknossos.datastore.storage.DataVaultService
 
 class DataStoreModule extends AbstractModule {
 
@@ -60,7 +60,6 @@ class DataStoreModule extends AbstractModule {
     bind(classOf[ZarrConnectomeFileService]).asEagerSingleton()
     bind(classOf[Hdf5ConnectomeFileService]).asEagerSingleton()
     bind(classOf[NeuroglancerPrecomputedMeshFileService]).asEagerSingleton()
-    bind(classOf[RemoteSourceDescriptorService]).asEagerSingleton()
     bind(classOf[ChunkCacheService]).asEagerSingleton()
     bind(classOf[DatasetCache]).asEagerSingleton()
   }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala

@@ -32,13 +32,13 @@ import com.scalableminds.webknossos.datastore.services.mesh.{
 }
 import com.scalableminds.webknossos.datastore.services.segmentindex.SegmentIndexFileService
 import com.scalableminds.webknossos.datastore.services.uploading._
-import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService
 import com.scalableminds.webknossos.datastore.services.connectome.{
   ByAgglomerateIdsRequest,
   BySynapseIdsRequest,
   SynapticPartnerDirection
 }
 import com.scalableminds.webknossos.datastore.services.mapping.AgglomerateService
+import com.scalableminds.webknossos.datastore.storage.DataVaultService
 import com.scalableminds.webknossos.datastore.slacknotification.DSSlackNotificationService
 import play.api.data.Form
 import play.api.data.Forms.{longNumber, nonEmptyText, number, tuple}
@@ -77,7 +77,7 @@ class DataSourceController @Inject()(
     fullMeshService: DSFullMeshService,
     uploadService: UploadService,
     meshFileService: MeshFileService,
-    remoteSourceDescriptorService: RemoteSourceDescriptorService,
+    dataVaultService: DataVaultService,
     val dsRemoteWebknossosClient: DSRemoteWebknossosClient,
     val dsRemoteTracingstoreClient: DSRemoteTracingstoreClient,
 )(implicit bodyParsers: PlayBodyParsers, ec: ExecutionContext)
@@ -682,7 +682,7 @@ class DataSourceController @Inject()(
       accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) {
         for {
           _ <- Fox.successful(())
-          pathsAllowed = request.body.map(remoteSourceDescriptorService.pathIsAllowedToAddDirectly)
+          pathsAllowed = request.body.map(dataVaultService.pathIsAllowedToAddDirectly)
           result = request.body.zip(pathsAllowed).map {
             case (path, isAllowed) => PathValidationResult(path, isAllowed)
           }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala

@@ -14,7 +14,7 @@ import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3Array
 import com.scalableminds.webknossos.datastore.datavault.VaultPath
 import com.scalableminds.webknossos.datastore.models.datasource.{DataFormat, DataSourceId, ElementClass, StaticLayer}
 import com.scalableminds.webknossos.datastore.models.requests.DataReadInstruction
-import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService
+import com.scalableminds.webknossos.datastore.storage.DataVaultService
 import com.typesafe.scalalogging.LazyLogging
 
 import scala.concurrent.duration._
@@ -24,7 +24,7 @@ import scala.concurrent.ExecutionContext
 
 class DatasetArrayBucketProvider(dataLayer: StaticLayer,
                                  dataSourceId: DataSourceId,
-                                 remoteSourceDescriptorServiceOpt: Option[RemoteSourceDescriptorService],
+                                 dataVaultServiceOpt: Option[DataVaultService],
                                  sharedChunkContentsCacheOpt: Option[AlfuCache[String, MultiArray]])
     extends BucketProvider
     with FoxImplicits
@@ -69,13 +69,12 @@ class DatasetArrayBucketProvider(dataLayer: StaticLayer,
     magLocatorOpt match {
       case None => Fox.empty
       case Some(magLocator) =>
-        remoteSourceDescriptorServiceOpt match {
-          case Some(remoteSourceDescriptorService: RemoteSourceDescriptorService) =>
+        dataVaultServiceOpt match {
+          case Some(dataVaultServiceOpt: DataVaultService) =>
             for {
-              magPath: VaultPath <- remoteSourceDescriptorService.vaultPathFor(readInstruction.baseDir,
-                                                                               readInstruction.dataSourceId,
-                                                                               readInstruction.dataLayer.name,
-                                                                               magLocator)
+              magPath: VaultPath <- dataVaultServiceOpt.vaultPathFor(magLocator,
+                                                                     readInstruction.dataSourceId,
+                                                                     readInstruction.dataLayer.name)
               chunkContentsCache <- sharedChunkContentsCacheOpt.toFox
               datasetArray <- dataLayer.dataFormat match {
                 case DataFormat.zarr =>

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/GoogleCloudDataVault.scala

@@ -4,7 +4,7 @@ import com.google.auth.oauth2.ServiceAccountCredentials
 import com.google.cloud.storage.{BlobId, BlobInfo, Storage, StorageException, StorageOptions}
 import com.scalableminds.util.accesscontext.TokenContext
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
-import com.scalableminds.webknossos.datastore.storage.{GoogleServiceAccountCredential, RemoteSourceDescriptor}
+import com.scalableminds.webknossos.datastore.storage.{GoogleServiceAccountCredential, CredentializedUPath}
 import com.scalableminds.util.tools.Box.tryo
 import com.scalableminds.webknossos.datastore.helpers.UPath
 import org.apache.commons.lang3.builder.HashCodeBuilder
@@ -114,8 +114,8 @@ class GoogleCloudDataVault(uri: URI, credential: Option[GoogleServiceAccountCred
 }
 
 object GoogleCloudDataVault {
-  def create(remoteSourceDescriptor: RemoteSourceDescriptor): GoogleCloudDataVault = {
-    val credential = remoteSourceDescriptor.credential.map(f => f.asInstanceOf[GoogleServiceAccountCredential])
-    new GoogleCloudDataVault(remoteSourceDescriptor.toUriUnsafe, credential)
+  def create(credentializedUpath: CredentializedUPath): GoogleCloudDataVault = {
+    val credential = credentializedUpath.credential.map(f => f.asInstanceOf[GoogleServiceAccountCredential])
+    new GoogleCloudDataVault(credentializedUpath.upath.toRemoteUriUnsafe, credential)
   }
 }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/HttpsDataVault.scala

@@ -7,7 +7,7 @@ import com.scalableminds.webknossos.datastore.storage.{
   DataVaultCredential,
   HttpBasicAuthCredential,
   LegacyDataVaultCredential,
-  RemoteSourceDescriptor
+  CredentializedUPath
 }
 import com.typesafe.scalalogging.LazyLogging
 import org.apache.commons.lang3.builder.HashCodeBuilder
@@ -158,11 +158,11 @@ object HttpsDataVault {
 
   /**
     * Factory method to create a new HttpsDataVault instance.
-    * @param remoteSourceDescriptor
+    * @param credentializedUpath
     * @param ws
     * @param dataStoreHost The host of the local data store that this vault is accessing. This is used to determine if a user token should be applied in requests.
     * @return
     */
-  def create(remoteSourceDescriptor: RemoteSourceDescriptor, ws: WSClient, dataStoreHost: String): HttpsDataVault =
-    new HttpsDataVault(remoteSourceDescriptor.credential, ws, dataStoreHost)
+  def create(credentializedUpath: CredentializedUPath, ws: WSClient, dataStoreHost: String): HttpsDataVault =
+    new HttpsDataVault(credentializedUpath.credential, ws, dataStoreHost)
 }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/S3DataVault.scala

@@ -4,7 +4,7 @@ import com.scalableminds.util.accesscontext.TokenContext
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.storage.{
   LegacyDataVaultCredential,
-  RemoteSourceDescriptor,
+  CredentializedUPath,
   S3AccessKeyCredential
 }
 import com.scalableminds.util.tools.Box.tryo
@@ -182,14 +182,13 @@ class S3DataVault(s3AccessKeyCredential: Option[S3AccessKeyCredential],
 }
 
 object S3DataVault {
-  def create(remoteSourceDescriptor: RemoteSourceDescriptor, ws: WSClient)(
-      implicit ec: ExecutionContext): S3DataVault = {
-    val credential = remoteSourceDescriptor.credential.flatMap {
+  def create(credentializedUpath: CredentializedUPath, ws: WSClient)(implicit ec: ExecutionContext): S3DataVault = {
+    val credential = credentializedUpath.credential.flatMap {
       case f: S3AccessKeyCredential     => Some(f)
       case f: LegacyDataVaultCredential => Some(f.toS3AccessKey)
       case _                            => None
     }
-    new S3DataVault(credential, remoteSourceDescriptor.toUriUnsafe, ws, ec)
+    new S3DataVault(credential, credentializedUpath.upath.toRemoteUriUnsafe, ws, ec)
   }
 
   def hostBucketFromUri(uri: URI): Option[String] = {

webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ExploreLocalLayerService.scala

@@ -6,7 +6,7 @@ import com.scalableminds.util.io.PathUtils
 import com.scalableminds.util.tools.{Fox, FoxImplicits, JsonHelper}
 import com.scalableminds.webknossos.datastore.datareaders.n5.N5Header
 import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceId, StaticLayer, UsableDataSource}
-import com.scalableminds.webknossos.datastore.storage.{DataVaultService, RemoteSourceDescriptor}
+import com.scalableminds.webknossos.datastore.storage.DataVaultService
 import com.scalableminds.util.tools.Box.tryo
 import com.scalableminds.webknossos.datastore.helpers.UPath
 import play.api.libs.json.Json
@@ -44,11 +44,10 @@ class ExploreLocalLayerService @Inject()(dataVaultService: DataVaultService)
       magDirectories <- tryo(Files.list(path.resolve(layerDirectory)).iterator().asScala.toList).toFox ?~> s"Could not resolve color directory as child of $path"
       layersWithVoxelSizes <- Fox.combined(magDirectories.map(dir =>
         for {
-          remoteSourceDescriptor <- Fox.successful(RemoteSourceDescriptor(UPath.fromLocalPath(dir), None))
           mag <- Vec3Int
             .fromMagLiteral(dir.getFileName.toString, allowScalar = true)
             .toFox ?~> s"invalid mag: ${dir.getFileName}"
-          vaultPath <- dataVaultService.getVaultPath(remoteSourceDescriptor) ?~> "dataVault.setup.failed"
+          vaultPath <- dataVaultService.vaultPathFor(dir) ?~> "dataVault.setup.failed"
           layersWithVoxelSizes <- new ZarrArrayExplorer(mag).explore(vaultPath, None)(TokenContext(None))
         } yield layersWithVoxelSizes))
       (layers, voxelSize) <- adaptLayersAndVoxelSize(layersWithVoxelSizes.flatten, None)
@@ -134,9 +133,7 @@ class ExploreLocalLayerService @Inject()(dataVaultService: DataVaultService)
         val subdirs = Files.list(path).iterator().asScala.toList
         if (subdirs.size == 1) subdirs.head.getFileName.toString else layerDirectory
       } else layerDirectory
-      fullPath = path.resolve(layer)
-      remoteSourceDescriptor <- Fox.successful(RemoteSourceDescriptor(UPath.fromLocalPath(fullPath), None))
-      vaultPath <- dataVaultService.getVaultPath(remoteSourceDescriptor) ?~> "dataVault.setup.failed"
+      vaultPath <- dataVaultService.vaultPathFor(path.resolve(layer)) ?~> "dataVault.setup.failed"
       layersWithVoxelSizes <- explorer.explore(vaultPath, None)(TokenContext(None))
       (layers, voxelSize) <- adaptLayersAndVoxelSize(layersWithVoxelSizes, None)
       relativeLayers = makeLayersRelative(layers)

webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ExploreRemoteLayerService.scala

@@ -9,7 +9,7 @@ import com.scalableminds.webknossos.datastore.datavault.VaultPath
 import com.scalableminds.webknossos.datastore.models.VoxelSize
 import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceId, StaticLayer, UsableDataSource}
 import com.scalableminds.webknossos.datastore.services.DSRemoteWebknossosClient
-import com.scalableminds.webknossos.datastore.storage.{DataVaultCredential, DataVaultService, RemoteSourceDescriptor}
+import com.scalableminds.webknossos.datastore.storage.{DataVaultCredential, DataVaultService, CredentializedUPath}
 import com.typesafe.scalalogging.LazyLogging
 import com.scalableminds.util.tools.{Box, Empty, Failure, Full}
 import com.scalableminds.webknossos.datastore.helpers.UPath
@@ -85,8 +85,7 @@ class ExploreRemoteLayerService @Inject()(dataVaultService: DataVaultService,
         .toFox ?~> s"Received invalid URI: $layerUri"
       _ <- assertLocalPathInWhitelist(upath)
       credentialOpt: Option[DataVaultCredential] <- Fox.runOptional(credentialId)(remoteWebknossosClient.getCredential)
-      remoteSource = RemoteSourceDescriptor(upath, credentialOpt)
-      remotePath <- dataVaultService.getVaultPath(remoteSource) ?~> "dataVault.setup.failed"
+      remotePath <- dataVaultService.vaultPathFor(CredentializedUPath(upath, credentialOpt)) ?~> "dataVault.setup.failed"
       layersWithVoxelSizes <- recursivelyExploreRemoteLayerAtPaths(
         List((remotePath, 0)),
         credentialId,