Add VFS support for tiledb:// URIs.

Author: teo-tsirpanis

tiledb/api/c_api/filesystem/filesystem_api_enum.h

@@ -40,4 +40,6 @@
     TILEDB_FILESYSTEM_ENUM(GCS) = 3,
     /** In-memory filesystem */
     TILEDB_FILESYSTEM_ENUM(MEMFS) = 4,
+    /** TileDB filesystem */
+    TILEDB_FILESYSTEM_ENUM(TILEDBFS) = 5,
 #endif

tiledb/sm/filesystem/failing_fs.h

@@ -0,0 +1,163 @@
+/**
+ * @file   failing_fs.h
+ *
+ * @section LICENSE
+ *
+ * The MIT License
+ *
+ * @copyright Copyright (c) 2025 TileDB, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ * @section DESCRIPTION
+ *
+ * This file defines the FailingFS class.
+ */
+
+#ifndef TILEDB_FAILINGFS_H
+#define TILEDB_FAILINGFS_h
+
+#include "tiledb/common/assert.h"
+#include "tiledb/sm/filesystem/filesystem_base.h"
+
+namespace tiledb::sm {
+/**
+ * Implements FilesystemBase by always throwing FilesystemException.
+ */
+class FailingFS : public FilesystemBase {
+ public:
+  /**
+   * Constructor.
+   * 
+   * @param uri_scheme The URI scheme this object will claim to support.
+   * @param message The message of the exceptions this object's operations will throw.
+   */
+  FailingFS(const std::string& uri_scheme, const std::string& message = "")
+      : uri_scheme_(uri_scheme)
+      , message_(message) {
+    iassert(!uri_scheme.empty());
+  }
+
+  bool supports_uri(const URI& uri) const override {
+    return uri.backend_name() == uri_scheme_;
+  }
+
+  virtual void create_dir(const URI&) const override {
+    throw get_exception();
+  }
+
+  virtual void touch(const URI&) const override {
+    throw get_exception();
+  }
+
+  virtual bool is_dir(const URI&) const override {
+    throw get_exception();
+  }
+
+  virtual bool is_file(const URI&) const override {
+    throw get_exception();
+  }
+
+  virtual void remove_dir(const URI&) const override {
+    throw get_exception();
+  }
+
+  virtual void remove_file(const URI&) const override {
+    throw get_exception();
+  }
+
+  virtual uint64_t file_size(const URI&) const override {
+    throw get_exception();
+  }
+
+  virtual std::vector<tiledb::common::filesystem::directory_entry>
+  ls_with_sizes(const URI&) const override {
+    throw get_exception();
+  }
+
+  virtual LsObjects ls_filtered(const URI&, ResultFilter, bool) const override {
+    throw get_exception();
+  }
+
+  virtual LsObjects ls_filtered_v2(
+      const URI&, ResultFilterV2, bool) const override {
+    throw get_exception();
+  }
+
+  virtual void move_file(const URI&, const URI&) const override {
+    throw get_exception();
+  }
+
+  virtual void move_dir(const URI&, const URI&) const override {
+    throw get_exception();
+  }
+
+  virtual void copy_file(const URI&, const URI&) const override {
+    throw get_exception();
+  }
+
+  virtual void copy_dir(const URI&, const URI&) const override {
+    throw get_exception();
+  }
+
+  virtual uint64_t read(const URI&, uint64_t, void*, uint64_t) override {
+    throw get_exception();
+  }
+
+  virtual void flush(const URI&, bool) override {
+    throw get_exception();
+  }
+
+  void sync(const URI&) const override {
+    throw get_exception();
+  }
+
+  virtual void write(const URI&, const void*, uint64_t, bool) override {
+    throw get_exception();
+  }
+
+  virtual bool is_bucket(const URI&) const override {
+    throw get_exception();
+  }
+
+  virtual bool is_empty_bucket(const URI&) const override {
+    throw get_exception();
+  }
+
+  virtual void create_bucket(const URI&) const override {
+    throw get_exception();
+  }
+
+  virtual void remove_bucket(const URI&) const override {
+    throw get_exception();
+  }
+
+  virtual void empty_bucket(const URI&) const override {
+    throw get_exception();
+  }
+
+ private:
+  const std::string uri_scheme_, message_;
+
+  FilesystemException get_exception() const {
+    return FilesystemException(message_);
+  }
+};
+}  // namespace tiledb::sm
+#endif  // TILEDB_FAILINGFS_H

tiledb/sm/filesystem/vfs.cc

@@ -38,6 +38,7 @@
 #include "tiledb/sm/buffer/buffer.h"
 #include "tiledb/sm/enums/filesystem.h"
 #include "tiledb/sm/enums/vfs_mode.h"
+#include "tiledb/sm/filesystem/failing_fs.h"
 #include "tiledb/sm/misc/parallel_functions.h"
 #include "tiledb/sm/misc/tdb_math.h"
 #include "tiledb/sm/stats/global_stats.h"
@@ -54,6 +55,39 @@ using namespace tiledb::sm::filesystem;
 
 namespace tiledb::sm {
 
+namespace {
+#ifdef HAVE_S3
+tdb_unique_ptr<FilesystemBase> make_tiledb_fs_failing(
+    const std::string& detail = "") {
+  return tdb_unique_ptr<FilesystemBase>(tdb_new(
+      FailingFS, "tiledb", "TileDB FS VFS not configured; " + detail));
+}
+
+tdb_unique_ptr<FilesystemBase> make_tiledbfs(
+    stats::Stats* parent_stats, ThreadPool* tp, const Config& config) {
+  auto rest_server =
+      config.get<std::string>("rest.server_address", Config::must_find);
+  if (rest_server.ends_with('/')) {
+    size_t pos = rest_server.find_last_not_of('/');
+    rest_server.resize(pos + 1);
+  }
+  if (rest_server.empty()) {
+    return make_tiledb_fs_failing("missing rest.server_address config option");
+  }
+  Config new_config{config};
+  throw_if_not_ok(new_config.set("vfs.s3.endpoint_override", rest_server + "/v4/files"));
+  auto token = config.get<std::string>("rest.token", Config::must_find);
+  if (!token.empty()) {
+    throw_if_not_ok(new_config.set("vfs.s3.aws_access_key_id", token));
+    throw_if_not_ok(new_config.set("vfs.s3.aws_secret_access_key", "unused"));
+  }
+  throw_if_not_ok(new_config.set("vfs.s3.use_virtual_addressing", "false"));
+  return tdb_unique_ptr<FilesystemBase>(
+      tdb_new(S3, parent_stats, tp, new_config));
+}
+#endif
+}  // namespace
+
 /* ********************************* */
 /*     CONSTRUCTORS & DESTRUCTORS    */
 /* ********************************* */
@@ -68,6 +102,10 @@ VFS::VFS(
     , Azure_within_VFS(io_tp, config)
     , GCS_within_VFS(io_tp, config)
     , S3_within_VFS(stats_, io_tp, config)
+    , local_(config)
+#ifdef HAVE_S3
+    , tiledbfs_(make_tiledbfs(parent_stats, io_tp, config))
+#endif
     , config_(config)
     , logger_(logger)
     , compute_tp_(compute_tp)
@@ -83,6 +121,7 @@ VFS::VFS(
 
   if constexpr (s3_enabled) {
     supported_fs_.insert(Filesystem::S3);
+    supported_fs_.insert(Filesystem::TILEDBFS);
   }
 
 #ifdef HAVE_AZURE
@@ -93,8 +132,6 @@ VFS::VFS(
   supported_fs_.insert(Filesystem::GCS);
 #endif
 
-  local_ = LocalFS(config_);
-
   supported_fs_.insert(Filesystem::MEMFS);
 }
 
@@ -130,6 +167,14 @@ const FilesystemBase& VFS::get_fs(const URI& uri) const {
   if (uri.is_memfs()) {
     return memfs_;
   }
+  if (uri.is_tiledb()) {
+#ifdef HAVE_S3
+    return *tiledbfs_;
+#else
+    throw VFSException(
+        "TileDB was built without S3 support, which is required for TileDB FS");
+#endif
+  }
   throw UnsupportedURI(uri.to_string());
 }
 

tiledb/sm/filesystem/vfs.h

@@ -1002,6 +1002,10 @@ class VFS : FilesystemBase,
 
   LocalFS local_;
 
+#if HAVE_S3
+  tdb_unique_ptr<FilesystemBase> tiledbfs_;
+#endif
+
   /** The in-memory filesystem which is always supported */
   MemFilesystem memfs_;