driverdog: Add ability to detect and decompress objects

The files that are linked at build time could be compressed which allows
space savings on disk for a minor boot time increase. driverdog now can
detect that the requested file may be compressed and have a .gz
extension. If so, then it will decompress before working with the file.

Signed-off-by: Matthew Yeazel <yeazelm@amazon.com>

Author: yeazelm

sources/Cargo.lock

@@ -131,7 +131,7 @@ derive-getters = "0.5"
 dns-lookup = "2"
 env_logger = "0.11"
 envy = "0.4"
-flate2 = { version = "1", default-features = false }
+flate2 = { version = "1" }
 fs2 = "0.4"
 futures = { version = "0.3", default-features = false }
 futures-channel = { version = "0.3", default-features = false }
@@ -193,7 +193,7 @@ syn = { version = "2", default-features = false }
 tar = { version = "0.4", default-features = false }
 tempfile = "3"
 test-case = "3"
-tokio = { version = "~1.43", default-features = false }  # LTS
+tokio = { version = "~1.43", default-features = false }            # LTS
 tokio-retry = "0.3"
 tokio-test = "0.4"
 tokio-tungstenite = { version = "0.20", default-features = false }

sources/Cargo.toml

@@ -10,6 +10,7 @@ exclude = ["README.md"]
 
 [dependencies]
 argh.workspace = true
+flate2.workspace = true
 log.workspace = true
 simplelog.workspace = true
 snafu.workspace = true

sources/driverdog/Cargo.toml

@@ -15,15 +15,22 @@ modes iterate over the `kernel-modules` and load them from that path with `modpr
 #[macro_use]
 extern crate log;
 
+// Standard library imports
+use std::{
+    collections::HashMap,
+    ffi::OsStr,
+    fs,
+    io::Read,
+    path::{Path, PathBuf},
+    process::{self, Command},
+};
+
+// External crate imports
 use argh::FromArgs;
+use flate2::read::GzDecoder;
 use serde::Deserialize;
 use simplelog::{Config as LogConfig, LevelFilter, SimpleLogger};
 use snafu::{ensure, OptionExt, ResultExt};
-use std::collections::HashMap;
-use std::ffi::OsStr;
-use std::fs;
-use std::path::{Path, PathBuf};
-use std::process::{self, Command};
 
 /// Path to the drivers configuration to use
 const DEFAULT_DRIVER_CONFIG_PATH: &str = "/etc/drivers/";
@@ -81,6 +88,50 @@ struct LinkModulesArgs {}
 #[argh(subcommand, name = "load-modules")]
 struct LoadModulesArgs {}
 
+/// Checks if there is a compressed file at the path provided and if so, provides the path with the extension
+pub(crate) fn is_compressed(file_path: &Path) -> Option<PathBuf> {
+    if !file_path.exists() {
+        let compressed_path = file_path.with_extension("o.gz");
+        if compressed_path.exists() {
+            return Some(compressed_path);
+        }
+    }
+    None
+}
+
+/// Decompresses a file to the requested destination. The original file is left
+/// intact since it could be on a read-only filesystem.
+fn decompress_and_copy(file_path: &Path, destination: &Path) -> Result<()> {
+    // Check if the file has .gz extension
+    if let Some(extension) = file_path.extension() {
+        if extension == "gz" {
+            // Read compressed content using streaming decompression
+            let compressed_file =
+                fs::File::open(file_path).context(error::OpenFileSnafu { path: file_path })?;
+            let mut decoder = GzDecoder::new(compressed_file);
+            let mut decompressed_content = Vec::new();
+
+            decoder
+                .read_to_end(&mut decompressed_content)
+                .context(error::DecompressSnafu {
+                    from: file_path,
+                    to: file_path,
+                })?;
+
+            // Write decompressed content to file without .gz extension
+            fs::write(destination, &decompressed_content)
+                .context(error::CreateFileSnafu { path: destination })?;
+
+            info!(
+                "Decompressed {} to {}",
+                file_path.display(),
+                destination.display()
+            );
+        }
+    }
+    Ok(())
+}
+
 #[derive(Deserialize, Debug)]
 #[serde(untagged)]
 /// Enum to hold the two types of configurations supported
@@ -219,10 +270,19 @@ where
         let object_file_path = build_dir.join(object_file);
         if !object_file_path.exists() {
             let from = driver_path.join(object_file);
-            fs::copy(&from, &object_file_path).context(error::CopySnafu {
-                from: &from,
-                to: &object_file_path,
-            })?;
+            match is_compressed(&from) {
+                Some(compressed_path) => {
+                    info!("Found compressed file: {:?}", compressed_path);
+                    decompress_and_copy(&compressed_path, &object_file_path)?;
+                }
+                None => {
+                    info!("Copying {:?}", &from);
+                    fs::copy(&from, &object_file_path).context(error::CopySnafu {
+                        from: &from,
+                        to: &object_file_path,
+                    })?;
+                }
+            }
         }
         dependencies_paths.push(object_file_path.to_string_lossy().into_owned());
     }
@@ -271,7 +331,7 @@ where
         .to_string_lossy()
         .into_owned();
     // Paths to the dependencies for this object file
-    let mut dependencies = object_file
+    let mut dependencies: Vec<String> = object_file
         .link_objects
         .iter()
         .map(|d| {
@@ -282,6 +342,21 @@ where
         })
         .collect();
 
+    for dependency in &mut dependencies {
+        let dep_path = PathBuf::from(dependency.as_str());
+        // If compressed, we need to decompress to a temporary path then shift the provided path to this new location
+        if let Some(compressed_path) = is_compressed(&dep_path) {
+            let uncompressed = build_dir.join(
+                dep_path
+                    .file_name()
+                    .context(error::InvalidFileNameSnafu { path: &dep_path })?,
+            );
+            info!("Decompressing to {:?}", &uncompressed);
+            decompress_and_copy(&compressed_path, &uncompressed)?;
+            *dependency = uncompressed.to_string_lossy().into_owned();
+        }
+    }
+
     // Link the object file
     let mut args = vec!["-r".to_string(), "-o".to_string(), object_path.clone()];
     args.append(&mut dependencies);
@@ -468,54 +543,105 @@ fn main() {
 }
 
 /// ＜コ：ミ くコ:彡 ＜コ：ミ くコ:彡 ＜コ：ミ くコ:彡 ＜コ：ミ くコ:彡 ＜コ：ミ くコ:彡 ＜コ：ミ くコ:彡
+/// Error types for driverdog operations
 mod error {
     use snafu::Snafu;
-    use std::path::PathBuf;
-    use std::process::{Command, Output};
+    use std::{
+        path::PathBuf,
+        process::{Command, Output},
+    };
 
     #[derive(Debug, Snafu)]
     #[snafu(visibility(pub(super)))]
     pub(super) enum Error {
-        #[snafu(display("'{}' failed - stderr: {}",
-                        bin_path, String::from_utf8_lossy(&output.stderr)))]
+        /// Command execution failed with non-zero exit status
+        #[snafu(display(
+            "'{}' failed - stderr: {}",
+            bin_path,
+            String::from_utf8_lossy(&output.stderr)
+        ))]
         CommandFailure { bin_path: String, output: Output },
 
-        #[snafu(display("Failed to copy '{}' to '{}': {}", from.display(), to.display(), source))]
+        /// Failed to execute command due to system error
+        #[snafu(display("Failed to execute '{:?}': {}", command, source))]
+        ExecutionFailure {
+            command: Box<Command>,
+            source: std::io::Error,
+        },
+
+        /// File copy operation failed
+        #[snafu(display(
+            "Failed to copy '{}' to '{}': {}",
+            from.display(),
+            to.display(),
+            source
+        ))]
         Copy {
             from: PathBuf,
             to: PathBuf,
             source: std::io::Error,
         },
 
-        #[snafu(display("Failed to deserialize '{}': {}", path.display(), source))]
-        Deserialize {
+        /// Failed to read from filesystem path
+        #[snafu(display("Failed to read path '{}': '{}'", path.display(), source))]
+        ReadPath {
             path: PathBuf,
-            source: toml::de::Error,
+            source: std::io::Error,
         },
 
-        #[snafu(display("Failed to execute '{:?}': {}", command, source))]
-        ExecutionFailure {
-            command: Box<Command>,
+        /// Failed to open file for reading
+        #[snafu(display("Failed to open file '{}': {}", path.display(), source))]
+        OpenFile {
+            path: PathBuf,
+            source: std::io::Error,
+        },
+
+        /// Failed to create or write file
+        #[snafu(display("Failed to create file '{}': {}", path.display(), source))]
+        CreateFile {
+            path: PathBuf,
             source: std::io::Error,
         },
 
+        /// TOML configuration deserialization failed
+        #[snafu(display("Failed to deserialize '{}': {}", path.display(), source))]
+        Deserialize {
+            path: PathBuf,
+            source: toml::de::Error,
+        },
+
+        /// Module path contains invalid UTF-8 characters.
         #[snafu(display("Module path '{}' is not UTF-8", path.display()))]
         InvalidModulePath { path: PathBuf },
 
+        /// Requested module set not found in configuration.
+        #[snafu(display("Missing module set '{}'", target))]
+        MissingModuleSet { target: String },
+
+        /// Logger initialization failed
         #[snafu(display("Failed to setup logger: {}", source))]
         Logger { source: log::SetLoggerError },
 
-        #[snafu(display("Missing module set '{}'", target))]
-        MissingModuleSet { target: String },
+        /// Temporary directory creation failed.
+        #[snafu(display("Failed to create temporary directory: {}", source))]
+        TmpDir { source: std::io::Error },
 
-        #[snafu(display("Failed to read path '{}': '{}'", path.display(), source))]
-        ReadPath {
-            path: PathBuf,
+        /// Gzip decompression operation failed.
+        #[snafu(display(
+            "Failed to decompress '{}' to '{}': {}",
+            from.display(),
+            to.display(),
+            source
+        ))]
+        Decompress {
+            from: PathBuf,
+            to: PathBuf,
             source: std::io::Error,
         },
 
-        #[snafu(display("Failed to create temporary directory: {}", source))]
-        TmpDir { source: std::io::Error },
+        /// File path has no filename component.
+        #[snafu(display("Path '{}' has no filename", path.display()))]
+        InvalidFileName { path: PathBuf },
     }
 }
 
@@ -524,13 +650,52 @@ type Result<T> = std::result::Result<T, error::Error>;
 #[cfg(test)]
 mod test {
     use super::*;
+    use std::fs::File;
     use std::path::PathBuf;
+    use tempfile::TempDir;
     use walkdir::WalkDir;
 
     fn test_data() -> PathBuf {
         PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src/tests")
     }
 
+    fn setup_compression_test_files() -> TempDir {
+        let temp_dir = tempfile::tempdir().expect("Failed to create temp directory");
+
+        let regular_file_path = temp_dir.path().join("module.o");
+        File::create(&regular_file_path).expect("Failed to create regular file");
+
+        let compressed_file_path = temp_dir.path().join("compressed_module.o.gz");
+        File::create(&compressed_file_path).expect("Failed to create compressed file");
+        temp_dir
+    }
+
+    #[test]
+    fn test_is_compressed_with_existing_file() {
+        let temp_dir = setup_compression_test_files();
+        let regular_file_path = temp_dir.path().join("module.o");
+        let result = is_compressed(&regular_file_path);
+        assert!(result.is_none());
+    }
+
+    #[test]
+    fn test_is_compressed_with_compressed_alternative() {
+        let temp_dir = setup_compression_test_files();
+        let non_existent_path = temp_dir.path().join("compressed_module.o");
+        let compressed_path = temp_dir.path().join("compressed_module.o.gz");
+        let result = is_compressed(&non_existent_path);
+        assert!(result.is_some());
+        assert_eq!(result.unwrap(), compressed_path);
+    }
+
+    #[test]
+    fn test_is_compressed_with_no_alternative() {
+        let temp_dir = setup_compression_test_files();
+        let non_existent_path = temp_dir.path().join("nonexistent.o");
+        let result = is_compressed(&non_existent_path);
+        assert!(result.is_none());
+    }
+
     #[test]
     fn parse_linking_config() {
         let driver_config_path = test_data();