Merge pull request #202 from DeterminateSystems/substitute-inputs-on-failure-only

Only try to substitute input if fetching from its original location fails

Author: edolstra

src/libfetchers/fetchers.cc

@@ -327,47 +327,59 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
     if (!scheme)
         throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
 
-    /* The tree may already be in the Nix store, or it could be
-       substituted (which is often faster than fetching from the
-       original source). So check that. We only do this for final
-       inputs, otherwise there is a risk that we don't return the
-       same attributes (like `lastModified`) that the "real" fetcher
-       would return.
-
-       FIXME: add a setting to disable this.
-       FIXME: substituting may be slower than fetching normally,
-       e.g. for fetchers like Git that are incremental!
-    */
-    if (isFinal() && getNarHash()) {
-        try {
-            auto storePath = computeStorePath(*store);
-
-            store->ensurePath(storePath);
+    std::optional<StorePath> storePath;
+    if (isFinal() && getNarHash())
+        storePath = computeStorePath(*store);
 
-            debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath));
+    auto makeStoreAccessor = [&]() -> std::pair<ref<SourceAccessor>, Input> {
+        auto accessor = make_ref<SubstitutedSourceAccessor>(makeStorePathAccessor(store, *storePath));
 
-            auto accessor = make_ref<SubstitutedSourceAccessor>(makeStorePathAccessor(store, storePath));
+        accessor->fingerprint = getFingerprint(store);
 
-            accessor->fingerprint = getFingerprint(store);
+        // FIXME: ideally we would use the `showPath()` of the
+        // "real" accessor for this fetcher type.
+        accessor->setPathDisplay("«" + to_string() + "»");
 
-            // FIXME: ideally we would use the `showPath()` of the
-            // "real" accessor for this fetcher type.
-            accessor->setPathDisplay("«" + to_string() + "»");
+        return {accessor, *this};
+    };
 
-            return {accessor, *this};
-        } catch (Error & e) {
-            debug("substitution of input '%s' failed: %s", to_string(), e.what());
-        }
+    /* If a tree with the expected hash is already in the Nix store,
+       reuse it. We only do this for final inputs, since otherwise
+       there is a risk that we don't return the same attributes (like
+       `lastModified`) that the "real" fetcher would return. */
+    if (storePath && store->isValidPath(*storePath)) {
+        debug("using input '%s' in '%s'", to_string(), store->printStorePath(*storePath));
+        return makeStoreAccessor();
     }
 
-    auto [accessor, result] = scheme->getAccessor(store, *this);
+    try {
+        auto [accessor, result] = scheme->getAccessor(store, *this);
 
-    if (!accessor->fingerprint)
-        accessor->fingerprint = result.getFingerprint(store);
-    else
-        result.cachedFingerprint = accessor->fingerprint;
+        if (!accessor->fingerprint)
+            accessor->fingerprint = result.getFingerprint(store);
+        else
+            result.cachedFingerprint = accessor->fingerprint;
 
-    return {accessor, std::move(result)};
+        return {accessor, std::move(result)};
+    } catch (Error & e) {
+        if (storePath) {
+            // Fall back to substitution.
+            try {
+                store->ensurePath(*storePath);
+                warn(
+                    "Successfully substituted input '%s' after failing to fetch it from its original location: %s",
+                    to_string(),
+                    e.info().msg);
+                return makeStoreAccessor();
+            }
+            // Ignore any substitution error, rethrow the original error.
+            catch (Error & e2) {
+                debug("substitution of input '%s' failed: %s", to_string(), e2.info().msg);
+            } catch (...) {
+            }
+        }
+        throw;
+    }
 }
 
 Input Input::applyOverrides(std::optional<std::string> ref, std::optional<Hash> rev) const

src/nix/main.cc

@@ -90,6 +90,7 @@ static void disableNet()
 {
     // FIXME: should check for command line overrides only.
     if (!settings.useSubstitutes.overridden)
+        // FIXME: should not disable local substituters (like file:///).
         settings.useSubstitutes = false;
     if (!settings.tarballTtl.overridden)
         settings.tarballTtl = std::numeric_limits<unsigned int>::max();

tests/functional/flakes/meson.build

@@ -35,6 +35,7 @@ suites += {
     'old-lockfiles.sh',
     'trace-ifd.sh',
     'build-time-flake-inputs.sh',
+    'substitution.sh',
   ],
   'workdir' : meson.current_source_dir(),
 }

tests/functional/flakes/substitution.sh

@@ -0,0 +1,31 @@
+#! /usr/bin/env bash
+
+# Test that inputs are substituted if they cannot be fetched from their original location.
+
+source ./common.sh
+
+if [[ $(nix config show lazy-trees) = true ]]; then
+    exit 0
+fi
+
+TODO_NixOS
+
+createFlake1
+createFlake2
+
+nix build --no-link "$flake2Dir#bar"
+
+path1="$(nix flake metadata --json "$flake1Dir" | jq -r .path)"
+
+# Building after an input disappeared should succeed, because it's still in the Nix store.
+mv "$flake1Dir" "$flake1Dir-tmp"
+nix build --no-link "$flake2Dir#bar" --no-eval-cache
+
+# Check that Nix will fall back to fetching the input from a substituter.
+cache="file://$TEST_ROOT/binary-cache"
+nix copy --to "$cache" "$path1"
+clearStore
+nix build --no-link "$flake2Dir#bar" --no-eval-cache --substitute --substituters "$cache"
+
+clearStore
+expectStderr 1 nix build --no-link "$flake2Dir#bar" --no-eval-cache | grepQuiet "The path.*does not exist"