Warn on startup when data dir is too small (#5601)

* Warn when data dir is too small

* Canonicalize mount points

Author: rdettai

quickwit/Cargo.lock

@@ -230,6 +230,7 @@ sqlx = { version = "0.7", features = [
 ] }
 syn = { version = "2.0.11", features = ["extra-traits", "full", "parsing"] }
 sync_wrapper = "0.1.2"
+sysinfo = "0.33.1"
 tabled = { version = "0.14", features = ["color"] }
 tempfile = "3"
 thiserror = "1"

quickwit/Cargo.toml

@@ -34,6 +34,7 @@ rayon = { workspace = true }
 regex = { workspace = true }
 serde = { workspace = true }
 siphasher = { workspace = true }
+sysinfo = { workspace = true }
 tempfile = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }

quickwit/quickwit-common/Cargo.toml

@@ -14,6 +14,8 @@
 
 use std::path::{Path, PathBuf};
 
+use bytesize::ByteSize;
+use sysinfo::{Disk, DiskRefreshKind};
 use tokio;
 
 /// Deletes the contents of a directory.
@@ -34,6 +36,38 @@ pub fn get_cache_directory_path(data_dir_path: &Path) -> PathBuf {
     data_dir_path.join("indexer-split-cache").join("splits")
 }
 
+/// Get the total size of the disk containing the given directory, or `None` if
+/// it couldn't be determined.
+pub fn get_disk_size(dir_path: &Path) -> Option<ByteSize> {
+    let disks = sysinfo::Disks::new_with_refreshed_list_specifics(
+        DiskRefreshKind::nothing().with_storage(),
+    );
+    let mut best_match: Option<(&Disk, PathBuf)> = None;
+    let dir_path = dir_path.canonicalize().ok()?;
+    for disk in disks.list() {
+        let canonical_mount_path = disk.mount_point().canonicalize().ok()?;
+        if dir_path.starts_with(&canonical_mount_path) {
+            match best_match {
+                Some((_, best_mount_point))
+                    if canonical_mount_path.starts_with(&best_mount_point) =>
+                {
+                    best_match = Some((disk, canonical_mount_path.clone()));
+                }
+                None => {
+                    best_match = Some((disk, canonical_mount_path.clone()));
+                }
+                _ => {}
+            }
+        }
+        if canonical_mount_path.starts_with(&dir_path) && canonical_mount_path != dir_path {
+            // if a disk is mounted within the directory, we can't determine the
+            // size of the directories disk
+            return None;
+        }
+    }
+    best_match.map(|(disk, _)| ByteSize::b(disk.total_space()))
+}
+
 #[cfg(test)]
 mod tests {
     use tempfile;

quickwit/quickwit-common/src/fs.rs

@@ -18,7 +18,9 @@ use std::str::FromStr;
 use std::time::Duration;
 
 use anyhow::{bail, Context};
+use bytesize::ByteSize;
 use http::HeaderMap;
+use quickwit_common::fs::get_disk_size;
 use quickwit_common::net::{find_private_ip, get_short_hostname, Host};
 use quickwit_common::new_coolid;
 use quickwit_common::uri::Uri;
@@ -338,9 +340,61 @@ fn validate(node_config: &NodeConfig) -> anyhow::Result<()> {
     if node_config.peer_seeds.is_empty() {
         warn!("peer seeds are empty");
     }
+    validate_disk_usage(node_config);
     Ok(())
 }
 
+/// A list of all the known disk budgets
+///
+/// External disk usage and unbounded disk usages, e.g the indexing workbench
+/// (indexing/) and the delete task workbench (delete_task_service/) are not included.
+#[derive(Default, Debug)]
+struct ExpectedDiskUsage {
+    // indexer / ingester
+    split_store_max_num_bytes: Option<ByteSize>,
+    max_queue_disk_usage: Option<ByteSize>,
+    // searcher
+    split_cache: Option<ByteSize>,
+}
+
+impl ExpectedDiskUsage {
+    fn from_config(node_config: &NodeConfig) -> Self {
+        let mut expected = Self::default();
+        if node_config.is_service_enabled(QuickwitService::Indexer) {
+            expected.max_queue_disk_usage =
+                Some(node_config.ingest_api_config.max_queue_disk_usage);
+            expected.split_store_max_num_bytes =
+                Some(node_config.indexer_config.split_store_max_num_bytes);
+        }
+        if node_config.is_service_enabled(QuickwitService::Searcher) {
+            expected.split_cache = node_config
+                .searcher_config
+                .split_cache
+                .map(|limits| limits.max_num_bytes);
+        }
+        expected
+    }
+
+    fn total(&self) -> ByteSize {
+        self.split_store_max_num_bytes.unwrap_or_default()
+            + self.max_queue_disk_usage.unwrap_or_default()
+            + self.split_cache.unwrap_or_default()
+    }
+}
+
+fn validate_disk_usage(node_config: &NodeConfig) {
+    if let Some(volume_size) = get_disk_size(&node_config.data_dir_path) {
+        let expected_disk_usage = ExpectedDiskUsage::from_config(node_config);
+        if expected_disk_usage.total() > volume_size {
+            warn!(
+                ?volume_size,
+                ?expected_disk_usage,
+                "data dir volume too small"
+            );
+        }
+    }
+}
+
 #[cfg(test)]
 impl Default for NodeConfigBuilder {
     fn default() -> Self {