Only try to substitute input if fetching from its original location fails

Previously we always tried to substitute first (by calling
`ensurePath()`). This wasn't a problem before lazy trees, because we
always end up copying to the store anyway. But that's not the case
with lazy trees. So frequently we ended up substituting an input that
we had already fetched.

This showed up as

  fetching source from https://cache.nixos.org

for inputs that you could swear Nix had already fetched just
before. This was especially a problem for Nixpkgs inputs, since many
Nixpkgs revisions are in cache.nixos.org.

Note that this could also be a problem without lazy trees, e.g. with a
local input (like a Nixpkgs clone) that happens to be in the binary
cache.

So we now only try substitution as a last resort, if we cannot fetch
the input normally.

Author: edolstra

src/libfetchers/fetchers.cc

@@ -327,47 +327,59 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> sto
     if (!scheme)
         throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
 
-    /* The tree may already be in the Nix store, or it could be
-       substituted (which is often faster than fetching from the
-       original source). So check that. We only do this for final
-       inputs, otherwise there is a risk that we don't return the
-       same attributes (like `lastModified`) that the "real" fetcher
-       would return.
-
-       FIXME: add a setting to disable this.
-       FIXME: substituting may be slower than fetching normally,
-       e.g. for fetchers like Git that are incremental!
-    */
-    if (isFinal() && getNarHash()) {
-        try {
-            auto storePath = computeStorePath(*store);
-
-            store->ensurePath(storePath);
+    std::optional<StorePath> storePath;
+    if (isFinal() && getNarHash())
+        storePath = computeStorePath(*store);
 
-            debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath));
+    auto makeStoreAccessor = [&]() -> std::pair<ref<SourceAccessor>, Input> {
+        auto accessor = make_ref<SubstitutedSourceAccessor>(makeStorePathAccessor(store, *storePath));
 
-            auto accessor = make_ref<SubstitutedSourceAccessor>(makeStorePathAccessor(store, storePath));
+        accessor->fingerprint = getFingerprint(store);
 
-            accessor->fingerprint = getFingerprint(store);
+        // FIXME: ideally we would use the `showPath()` of the
+        // "real" accessor for this fetcher type.
+        accessor->setPathDisplay("«" + to_string() + "»");
 
-            // FIXME: ideally we would use the `showPath()` of the
-            // "real" accessor for this fetcher type.
-            accessor->setPathDisplay("«" + to_string() + "»");
+        return {accessor, *this};
+    };
 
-            return {accessor, *this};
-        } catch (Error & e) {
-            debug("substitution of input '%s' failed: %s", to_string(), e.what());
-        }
+    /* If a tree with the expected hash is already in the Nix store,
+       reuse it. We only do this for final inputs, since otherwise
+       there is a risk that we don't return the same attributes (like
+       `lastModified`) that the "real" fetcher would return. */
+    if (storePath && store->isValidPath(*storePath)) {
+        debug("using input '%s' in '%s'", to_string(), store->printStorePath(*storePath));
+        return makeStoreAccessor();
     }
 
-    auto [accessor, result] = scheme->getAccessor(store, *this);
+    try {
+        auto [accessor, result] = scheme->getAccessor(store, *this);
 
-    if (!accessor->fingerprint)
-        accessor->fingerprint = result.getFingerprint(store);
-    else
-        result.cachedFingerprint = accessor->fingerprint;
+        if (!accessor->fingerprint)
+            accessor->fingerprint = result.getFingerprint(store);
+        else
+            result.cachedFingerprint = accessor->fingerprint;
 
-    return {accessor, std::move(result)};
+        return {accessor, std::move(result)};
+    } catch (Error & e) {
+        if (storePath) {
+            // Fall back to substitution.
+            try {
+                store->ensurePath(*storePath);
+                warn(
+                    "Successfully substituted input '%s' after failing to fetch it from its original location: %s",
+                    to_string(),
+                    e.info().msg);
+                return makeStoreAccessor();
+            }
+            // Ignore any substitution error, rethrow the original error.
+            catch (Error & e2) {
+                debug("substitution of input '%s' failed: %s", to_string(), e2.info().msg);
+            } catch (...) {
+            }
+        }
+        throw;
+    }
 }
 
 Input Input::applyOverrides(std::optional<std::string> ref, std::optional<Hash> rev) const

src/nix/main.cc

@@ -90,6 +90,7 @@ static void disableNet()
 {
     // FIXME: should check for command line overrides only.
     if (!settings.useSubstitutes.overridden)
+        // FIXME: should not disable local substituters (like file:///).
         settings.useSubstitutes = false;
     if (!settings.tarballTtl.overridden)
         settings.tarballTtl = std::numeric_limits<unsigned int>::max();

tests/functional/flakes/meson.build

@@ -35,6 +35,7 @@ suites += {
     'old-lockfiles.sh',
     'trace-ifd.sh',
     'build-time-flake-inputs.sh',
+    'substitution.sh',
   ],
   'workdir' : meson.current_source_dir(),
 }

tests/functional/flakes/substitution.sh

@@ -0,0 +1,31 @@
+#! /usr/bin/env bash
+
+# Test that inputs are substituted if they cannot be fetched from their original location.
+
+source ./common.sh
+
+if [[ $(nix config show lazy-trees) = true ]]; then
+    exit 0
+fi
+
+TODO_NixOS
+
+createFlake1
+createFlake2
+
+nix build --no-link "$flake2Dir#bar"
+
+path1="$(nix flake metadata --json "$flake1Dir" | jq -r .path)"
+
+# Building after an input disappeared should succeed, because it's still in the Nix store.
+mv "$flake1Dir" "$flake1Dir-tmp"
+nix build --no-link "$flake2Dir#bar" --no-eval-cache
+
+# Check that Nix will fall back to fetching the input from a substituter.
+cache="file://$TEST_ROOT/binary-cache"
+nix copy --to "$cache" "$path1"
+clearStore
+nix build --no-link "$flake2Dir#bar" --no-eval-cache --substitute --substituters "$cache"
+
+clearStore
+expectStderr 1 nix build --no-link "$flake2Dir#bar" --no-eval-cache | grepQuiet "The path.*does not exist"