Refactoring apiclient apply's get logic and apiclient apply support for S3, SSM, and Secrets Manager URIs

sources/Cargo.toml

@@ -114,6 +114,9 @@ aws-lc-rs = "1"
 aws-sdk-cloudformation = "1"
 aws-sdk-ec2 = "1"
 aws-sdk-eks = "1"
+aws-sdk-s3 = "1"
+aws-sdk-secretsmanager = "1"
+aws-sdk-ssm = "1"
 aws-smithy-runtime = "1"
 aws-smithy-runtime-api = "1"
 aws-smithy-types = "1"

sources/api/apiclient/Cargo.toml

@@ -15,11 +15,17 @@ tls = ["dep:rustls", "dep:aws-lc-rs", "reqwest/rustls-tls-native-roots"]
 fips = ["tls", "aws-lc-rs/fips", "rustls/fips"]
 
 [dependencies]
+async-trait.workspace = true
+aws-config.workspace = true
 aws-lc-rs = { workspace = true, optional = true, features = ["bindgen"] }
+aws-sdk-s3.workspace = true
+aws-sdk-secretsmanager.workspace = true
+aws-sdk-ssm.workspace = true
 base64.workspace = true
 constants.workspace = true
 datastore.workspace = true
 futures.workspace = true
+futures-util.workspace = true
 futures-channel.workspace = true
 http.workspace = true
 httparse.workspace = true
@@ -47,3 +53,7 @@ url.workspace = true
 
 [build-dependencies]
 generate-readme.workspace = true
+
+[dev-dependencies]
+tempfile.workspace = true
+test-case.workspace = true

sources/api/apiclient/src/apply.rs

@@ -1,15 +1,15 @@
 //! This module allows application of settings from URIs or stdin.  The inputs are expected to be
 //! TOML settings files, in the same format as user data, or the JSON equivalent.  The inputs are
 //! pulled and applied to the API server in a single transaction.
-
+use crate::apply::error::ResolverFailureSnafu;
 use crate::rando;
-use futures::future::{join, ready, TryFutureExt};
+use futures::future::{join, ready};
 use futures::stream::{self, StreamExt};
 use reqwest::Url;
 use serde::de::{Deserialize, IntoDeserializer};
-use snafu::{futures::try_future::TryFutureExt as SnafuTryFutureExt, OptionExt, ResultExt};
+use snafu::{OptionExt, ResultExt};
+use std::convert::TryFrom;
 use std::path::Path;
-use tokio::io::AsyncReadExt;
 
 /// Reads settings in TOML or JSON format from files at the requested URIs (or from stdin, if given
 /// "-"), then commits them in a single transaction and applies them to the system.
@@ -66,47 +66,67 @@ where
     Ok(())
 }
 
+/// Holds the raw input string and the URL (if it parses).
+pub struct SettingsInput {
+    pub input: String,
+    pub parsed_url: Option<Url>,
+}
+impl SettingsInput {
+    pub(crate) fn new(input: impl Into<String>) -> Self {
+        let input = input.into();
+        let parsed_url = match Url::parse(&input) {
+            Ok(url) => Some(url),
+            Err(err) => {
+                log::debug!("URL parse failed for '{}': {}", input, err);
+                None
+            }
+        };
+        SettingsInput { input, parsed_url }
+    }
+}
+
 /// Retrieves the given source location and returns the result in a String.
 async fn get<S>(input_source: S) -> Result<String>
 where
-    S: Into<String>,
+    S: AsRef<str>,
 {
-    let input_source = input_source.into();
-
-    // Read from stdin if "-" was given.
-    if input_source == "-" {
-        let mut output = String::new();
-        tokio::io::stdin()
-            .read_to_string(&mut output)
-            .context(error::StdinReadSnafu)
-            .await?;
-        return Ok(output);
-    }
+    let settings = SettingsInput::new(input_source.as_ref());
+    let resolver = select_resolver(&settings)?;
+    resolver.resolve().await.context(ResolverFailureSnafu)
+}
 
-    // Otherwise, the input should be a URI; parse it to know what kind.
-    // Until reqwest handles file:// URIs: https://github.com/seanmonstar/reqwest/issues/178
-    let uri = Url::parse(&input_source).context(error::UriSnafu {
-        input_source: &input_source,
-    })?;
-    if uri.scheme() == "file" {
-        // Turn the URI to a file path, and return a future that reads it.
-        let path = uri.to_file_path().ok().context(error::FileUriSnafu {
-            input_source: &input_source,
-        })?;
-        tokio::fs::read_to_string(path)
-            .context(error::FileReadSnafu { input_source })
-            .await
-    } else {
-        // Return a future that contains the text of the (non-file) URI.
-        reqwest::get(uri)
-            .and_then(|response| ready(response.error_for_status()))
-            .and_then(|response| response.text())
-            .context(error::ReqwestSnafu {
-                uri: input_source,
-                method: "GET",
-            })
-            .await
+/// Macro to try multiple settings resolver types in sequence, returning the first one that succeeds.
+macro_rules! try_resolvers {
+    ($input:expr, $($resolver_type:ty),+ $(,)?) => {
+        $(
+            if let Ok(r) = <$resolver_type>::try_from($input) {
+                log::debug!("select_resolver: picked {}", stringify!($resolver_type));
+                return Ok(Box::new(r));
+            }
+        )+
+    };
+}
+
+/// Choose which UriResolver applies to `input` (stdin, file://, http(s)://, s3://, secretsmanager://, and ssm://).
+fn select_resolver(input: &SettingsInput) -> Result<Box<dyn crate::uri_resolver::UriResolver>> {
+    use crate::uri_resolver::*;
+
+    try_resolvers!(
+        input,
+        StdinUri,
+        FileUri,
+        HttpUri,
+        S3Uri,
+        SecretsManagerArn,
+        SecretsManagerUri,
+        SsmArn,
+        SsmUri,
+    );
+
+    error::NoResolverSnafu {
+        input_source: input.input.clone(),
     }
+    .fail()
 }
 
 /// Takes a string of TOML or JSON settings data and reserializes
@@ -155,14 +175,8 @@ mod error {
             source: Box<crate::Error>,
         },
 
-        #[snafu(display("Failed to read given file '{}': {}", input_source, source))]
-        FileRead {
-            input_source: String,
-            source: std::io::Error,
-        },
-
-        #[snafu(display("Given invalid file URI '{}'", input_source))]
-        FileUri { input_source: String },
+        #[snafu(display("No URI resolver found for '{}'", input_source))]
+        NoResolver { input_source: String },
 
         #[snafu(display(
             "Input '{}' is not valid TOML or JSON.  (TOML error: {})  (JSON error: {})",
@@ -218,25 +232,54 @@ mod error {
             source: reqwest::Error,
         },
 
-        #[snafu(display("Failed to read standard input: {}", source))]
-        StdinRead { source: std::io::Error },
-
         #[snafu(display(
             "Failed to translate TOML from '{}' to JSON for API: {}",
             input_source,
             source
         ))]
         TomlToJson {
             input_source: String,
-            source: toml::de::Error,
+            #[snafu(source(from(toml::de::Error, Box::new)))]
+            source: Box<toml::de::Error>,
         },
 
         #[snafu(display("Given invalid URI '{}': {}", input_source, source))]
         Uri {
             input_source: String,
             source: url::ParseError,
         },
+
+        #[snafu(display("Resolver failed: {}", source))]
+        ResolverFailure {
+            #[snafu(source(from(crate::uri_resolver::ResolverError, Box::new)))]
+            source: Box<crate::uri_resolver::ResolverError>,
+        },
     }
 }
 pub use error::Error;
 pub type Result<T> = std::result::Result<T, error::Error>;
+
+#[cfg(test)]
+mod resolver_selection_tests {
+    use super::select_resolver;
+    use crate::apply::SettingsInput;
+    use std::any::{Any, TypeId};
+    use test_case::test_case;
+
+    #[test_case("-",                                                                TypeId::of::<crate::uri_resolver::StdinUri>();                  "stdin")]
+    #[test_case("file:///tmp/folder",                                               TypeId::of::<crate::uri_resolver::FileUri>();                   "file")]
+    #[test_case("http://amazon.com",                                                TypeId::of::<crate::uri_resolver::HttpUri>();                   "http")]
+    #[test_case("https://amazon.com",                                               TypeId::of::<crate::uri_resolver::HttpUri>();                   "https")]
+    #[test_case("s3://mybucket/path",                                               TypeId::of::<crate::uri_resolver::S3Uri>();                     "s3")]
+    #[test_case("secretsmanager://sec",                                             TypeId::of::<crate::uri_resolver::SecretsManagerUri>();         "secrets")]
+    #[test_case("ssm://param",                                                      TypeId::of::<crate::uri_resolver::SsmUri>();                    "ssmUri")]
+    #[test_case("arn:aws:ssm:<region>:<account_id>:parameter/<name>",               TypeId::of::<crate::uri_resolver::SsmArn>();                    "ssmArn")]
+    #[test_case("arn:aws:secretsmanager:<region>:<account-id>:secret:<secret-id>",  TypeId::of::<crate::uri_resolver::SecretsManagerArn>();         "secretsmanagerArn")]
+
+    fn resolver_selection(input: &str, expected: std::any::TypeId) {
+        let settings = SettingsInput::new(input);
+        let resolver = select_resolver(&settings).expect("should have a resolver for this scheme");
+        let any = resolver.as_ref() as &dyn Any;
+        assert_eq!(any.type_id(), expected);
+    }
+}

sources/api/apiclient/src/lib.rs

@@ -27,6 +27,7 @@ pub mod reboot;
 pub mod report;
 pub mod set;
 pub mod update;
+pub mod uri_resolver;
 
 mod error {
     use snafu::Snafu;