diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionInputPrefetcher.java b/src/main/java/com/google/devtools/build/lib/actions/ActionInputPrefetcher.java index 92c6325c212bac..be64ef5e162715 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ActionInputPrefetcher.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionInputPrefetcher.java @@ -13,12 +13,15 @@ // limitations under the License. package com.google.devtools.build.lib.actions; +import java.io.IOException; + /** Prefetches files to local disk. */ public interface ActionInputPrefetcher { public static final ActionInputPrefetcher NONE = new ActionInputPrefetcher() { @Override - public void prefetchFiles(Iterable input) { + public void prefetchFiles(Iterable inputs, + MetadataProvider metadataProvider) { // Do nothing. } }; @@ -28,5 +31,5 @@ public void prefetchFiles(Iterable input) { * *

For any path not under this prefetcher's control, the call should be a no-op. */ - void prefetchFiles(Iterable input); + void prefetchFiles(Iterable inputs, MetadataProvider metadataProvider) throws IOException, InterruptedException; } diff --git a/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java index e95413b00f4e58..4834abfd229da5 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java @@ -181,11 +181,11 @@ public int getId() { } @Override - public void prefetchInputs() throws IOException { + public void prefetchInputs() throws IOException, InterruptedException { if (Spawns.shouldPrefetchInputsForLocalExecution(spawn)) { actionExecutionContext .getActionInputPrefetcher() - .prefetchFiles(getInputMapping(true).values()); + .prefetchFiles(getInputMapping(true).values(), getMetadataProvider()); } } diff --git a/src/main/java/com/google/devtools/build/lib/exec/SpawnRunner.java b/src/main/java/com/google/devtools/build/lib/exec/SpawnRunner.java index 0d70563e407ebc..fc4d10172a67d2 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/SpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/exec/SpawnRunner.java @@ -13,7 +13,9 @@ // limitations under the License. package com.google.devtools.build.lib.exec; +import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander; import com.google.devtools.build.lib.actions.ArtifactPathResolver; import com.google.devtools.build.lib.actions.ExecException; @@ -25,6 +27,7 @@ import com.google.devtools.build.lib.vfs.PathFragment; import java.io.IOException; import java.time.Duration; +import java.util.Collection; import java.util.SortedMap; /** @@ -129,6 +132,7 @@ public enum ProgressStatus { * by different threads, so they MUST not call any shared non-thread-safe objects. */ interface SpawnExecutionContext { + /** * Returns a unique id for this spawn, to be used for logging. Note that a single spawn may be * passed to multiple {@link SpawnRunner} implementations, so any log entries should also @@ -137,25 +141,15 @@ interface SpawnExecutionContext { int getId(); /** - * Prefetches the Spawns input files to the local machine. There are cases where Bazel runs on a - * network file system, and prefetching the files in parallel is a significant performance win. - * This should only be called by local strategies when local execution is imminent. - * - *

Should be called with the equivalent of: - * - * policy.prefetchInputs( - * Iterables.filter(policy.getInputMapping().values(), Predicates.notNull())); - * + * Prefetches the {@link Spawn}'s input files to the local machine. * - *

Note in particular that {@link #getInputMapping} may return {@code null} values, but - * this method does not accept {@code null} values. + *

This is used by remote caching / execution to only download remote build outputs to the + * local machine that are strictly required. It's also used as a performance optimization in + * cases when Bazel runs on a network filesystem. * - *

The reason why this method requires passing in the inputs is that getInputMapping may be - * slow to compute, so if the implementation already called it, we don't want to compute it - * again. I suppose we could require implementations to memoize getInputMapping (but not compute - * it eagerly), and that may change in the future. + *

This should only be called by local strategies when local execution is imminent. */ - void prefetchInputs() throws IOException; + void prefetchInputs() throws IOException, InterruptedException; /** * The input file metadata cache for this specific spawn, which can be used to efficiently @@ -199,6 +193,18 @@ SortedMap getInputMapping(boolean expandTreeArtifacts /** Reports a progress update to the Spawn strategy. */ void report(ProgressStatus state, String name); + + /** + * Returns the collection of files that this command must write and make available via + * the local {@link com.google.devtools.build.lib.vfs.FileSystem}. The returned output + * artifacts are a subset of the action's output artifacts. + * + *

This is for use with remote execution, where as an optimization we don't want to + * download all output files. + */ + default Collection getRequiredLocalOutputs() { + return ImmutableList.of(); + } } /**