diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionInputPrefetcher.java b/src/main/java/com/google/devtools/build/lib/actions/ActionInputPrefetcher.java index 92c6325c212bac..be64ef5e162715 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ActionInputPrefetcher.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionInputPrefetcher.java @@ -13,12 +13,15 @@ // limitations under the License. package com.google.devtools.build.lib.actions; +import java.io.IOException; + /** Prefetches files to local disk. */ public interface ActionInputPrefetcher { public static final ActionInputPrefetcher NONE = new ActionInputPrefetcher() { @Override - public void prefetchFiles(Iterable extends ActionInput> input) { + public void prefetchFiles(Iterable extends ActionInput> inputs, + MetadataProvider metadataProvider) { // Do nothing. } }; @@ -28,5 +31,5 @@ public void prefetchFiles(Iterable extends ActionInput> input) { * *
For any path not under this prefetcher's control, the call should be a no-op. */ - void prefetchFiles(Iterable extends ActionInput> input); + void prefetchFiles(Iterable extends ActionInput> inputs, MetadataProvider metadataProvider) throws IOException, InterruptedException; } diff --git a/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java index e95413b00f4e58..4834abfd229da5 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java @@ -181,11 +181,11 @@ public int getId() { } @Override - public void prefetchInputs() throws IOException { + public void prefetchInputs() throws IOException, InterruptedException { if (Spawns.shouldPrefetchInputsForLocalExecution(spawn)) { actionExecutionContext .getActionInputPrefetcher() - .prefetchFiles(getInputMapping(true).values()); + .prefetchFiles(getInputMapping(true).values(), getMetadataProvider()); } } diff --git a/src/main/java/com/google/devtools/build/lib/exec/SpawnRunner.java b/src/main/java/com/google/devtools/build/lib/exec/SpawnRunner.java index 0d70563e407ebc..fc4d10172a67d2 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/SpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/exec/SpawnRunner.java @@ -13,7 +13,9 @@ // limitations under the License. package com.google.devtools.build.lib.exec; +import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander; import com.google.devtools.build.lib.actions.ArtifactPathResolver; import com.google.devtools.build.lib.actions.ExecException; @@ -25,6 +27,7 @@ import com.google.devtools.build.lib.vfs.PathFragment; import java.io.IOException; import java.time.Duration; +import java.util.Collection; import java.util.SortedMap; /** @@ -129,6 +132,7 @@ public enum ProgressStatus { * by different threads, so they MUST not call any shared non-thread-safe objects. */ interface SpawnExecutionContext { + /** * Returns a unique id for this spawn, to be used for logging. Note that a single spawn may be * passed to multiple {@link SpawnRunner} implementations, so any log entries should also @@ -137,25 +141,15 @@ interface SpawnExecutionContext { int getId(); /** - * Prefetches the Spawns input files to the local machine. There are cases where Bazel runs on a - * network file system, and prefetching the files in parallel is a significant performance win. - * This should only be called by local strategies when local execution is imminent. - * - *
Should be called with the equivalent of:
- *
- * policy.prefetchInputs(
- * Iterables.filter(policy.getInputMapping().values(), Predicates.notNull()));
- *
+ * Prefetches the {@link Spawn}'s input files to the local machine.
*
- *
Note in particular that {@link #getInputMapping} may return {@code null} values, but - * this method does not accept {@code null} values. + *
This is used by remote caching / execution to only download remote build outputs to the + * local machine that are strictly required. It's also used as a performance optimization in + * cases when Bazel runs on a network filesystem. * - *
The reason why this method requires passing in the inputs is that getInputMapping may be - * slow to compute, so if the implementation already called it, we don't want to compute it - * again. I suppose we could require implementations to memoize getInputMapping (but not compute - * it eagerly), and that may change in the future. + *
This should only be called by local strategies when local execution is imminent.
*/
- void prefetchInputs() throws IOException;
+ void prefetchInputs() throws IOException, InterruptedException;
/**
* The input file metadata cache for this specific spawn, which can be used to efficiently
@@ -199,6 +193,18 @@ SortedMap This is for use with remote execution, where as an optimization we don't want to
+ * download all output files.
+ */
+ default Collection