Rename and add tests for util/InputIndexer and init PersistentExecuto…

…rTest

src/main/java/build/buildfarm/worker/persistent/WorkFilesContext.java

@@ -1,7 +1,7 @@
 package build.buildfarm.worker.persistent;
 
 import build.buildfarm.v1test.Tree;
-import build.buildfarm.worker.util.TreeWalker;
+import build.buildfarm.worker.util.InputsIndexer;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.devtools.build.lib.worker.WorkerProtocol.Input;
@@ -19,7 +19,7 @@ public class WorkFilesContext {
 
   public final ImmutableList<String> outputDirectories;
 
-  private final TreeWalker treeWalker;
+  private final InputsIndexer inputsIndexer;
 
   private ImmutableMap<Path, Input> pathInputs = null;
 
@@ -37,14 +37,14 @@ public WorkFilesContext(
     this.outputFiles = outputFiles;
     this.outputDirectories = outputDirectories;
 
-    this.treeWalker = new TreeWalker(execTree);
+    this.inputsIndexer = new InputsIndexer(execTree);
   }
 
   // Paths are absolute paths from the opRoot; same as the Input.getPath();
   public ImmutableMap<Path, Input> getPathInputs() {
     synchronized (this) {
       if (pathInputs == null) {
-        pathInputs = treeWalker.getAllInputs(opRoot);
+        pathInputs = inputsIndexer.getAllInputs(opRoot);
       }
     }
     return pathInputs;
@@ -53,7 +53,7 @@ public ImmutableMap<Path, Input> getPathInputs() {
   public ImmutableMap<Path, Input> getToolInputs() {
     synchronized (this) {
       if (toolInputs == null) {
-        toolInputs = treeWalker.getToolInputs(opRoot);
+        toolInputs = inputsIndexer.getToolInputs(opRoot);
       }
     }
     return toolInputs;

src/main/java/build/buildfarm/worker/util/TreeWalker.java → src/main/java/build/buildfarm/worker/util/InputsIndexer.java

@@ -15,8 +15,10 @@
 /**
  * Organizes action Inputs into files, extracting their paths, and differentiates tool inputs (e.g.
  * JavaBuilder, Scalac, etc.)
+ *
+ * <p>Indexes (and partitions) Inputs from an action's Merkle Tree.
  */
-public class TreeWalker {
+public class InputsIndexer {
   // See: https://github.com/bazelbuild/bazel/issues/10091
   public static final String BAZEL_TOOL_INPUT_MARKER = "bazel_tool_input";
 
@@ -27,7 +29,7 @@ public class TreeWalker {
   ImmutableMap<Path, Input> absPathInputs = null;
   ImmutableMap<Path, Input> toolInputs = null;
 
-  public TreeWalker(Tree tree) {
+  public InputsIndexer(Tree tree) {
     this.tree = tree;
     this.proxyDirs = new ProxyDirectoriesIndex(tree.getDirectoriesMap());
   }

src/test/java/build/buildfarm/worker/persistent/BUILD

@@ -0,0 +1,31 @@
+java_test(
+    name = "tests",
+    size = "small",
+    srcs = glob(["*.java"]),
+    test_class = "build.buildfarm.AllTests",
+    deps = [
+        "//src/main/java/build/buildfarm/common",
+        "//src/main/java/build/buildfarm/common/config",
+        "//src/main/java/build/buildfarm/instance",
+        "//src/main/java/build/buildfarm/worker",
+        "//src/main/java/build/buildfarm/worker/resources",
+        "//src/main/protobuf:build_buildfarm_v1test_buildfarm_java_proto",
+        "//src/test/java/build/buildfarm:test_runner",
+        "@bazel_tools//src/main/protobuf:worker_protocol_java_proto",
+        "@googleapis//:google_rpc_code_java_proto",
+        "@maven//:com_github_jnr_jnr_constants",
+        "@maven//:com_github_jnr_jnr_ffi",
+        "@maven//:com_github_serceman_jnr_fuse",
+        "@maven//:com_google_guava_guava",
+        "@maven//:com_google_jimfs_jimfs",
+        "@maven//:com_google_protobuf_protobuf_java",
+        "@maven//:com_google_truth_truth",
+        "@maven//:io_grpc_grpc_api",
+        "@maven//:io_grpc_grpc_context",
+        "@maven//:io_grpc_grpc_core",
+        "@maven//:io_grpc_grpc_protobuf",
+        "@maven//:org_mockito_mockito_core",
+        "@maven//:org_projectlombok_lombok",
+        "@remote_apis//:build_bazel_remote_execution_v2_remote_execution_java_proto",
+    ],
+)

src/test/java/build/buildfarm/worker/persistent/PersistentExecutorTest.java

@@ -0,0 +1,13 @@
+package build.buildfarm.worker.persistent;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class PersistentExecutorTest {
+  @Test
+  public void testProtoCoordinatorCreatesDirs() throws Exception {
+    assert (true);
+  }
+}

src/test/java/build/buildfarm/worker/util/BUILD

@@ -0,0 +1,30 @@
+java_test(
+    name = "tests",
+    size = "small",
+    srcs = glob(["*.java"]),
+    test_class = "build.buildfarm.AllTests",
+    deps = [
+        "//src/main/java/build/buildfarm/cas",
+        "//src/main/java/build/buildfarm/common",
+        "//src/main/java/build/buildfarm/common/config",
+        "//src/main/java/build/buildfarm/worker/util",
+        "//src/main/protobuf:build_buildfarm_v1test_buildfarm_java_proto",
+        "//src/test/java/build/buildfarm:test_runner",
+        "@bazel_tools//src/main/protobuf:worker_protocol_java_proto",
+        "@googleapis//:google_rpc_code_java_proto",
+        "@maven//:com_github_jnr_jnr_constants",
+        "@maven//:com_github_jnr_jnr_ffi",
+        "@maven//:com_github_serceman_jnr_fuse",
+        "@maven//:com_google_guava_guava",
+        "@maven//:com_google_jimfs_jimfs",
+        "@maven//:com_google_protobuf_protobuf_java",
+        "@maven//:com_google_truth_truth",
+        "@maven//:io_grpc_grpc_api",
+        "@maven//:io_grpc_grpc_context",
+        "@maven//:io_grpc_grpc_core",
+        "@maven//:io_grpc_grpc_protobuf",
+        "@maven//:org_mockito_mockito_core",
+        "@maven//:org_projectlombok_lombok",
+        "@remote_apis//:build_bazel_remote_execution_v2_remote_execution_java_proto",
+    ],
+)

src/test/java/build/buildfarm/worker/util/InputsIndexerTest.java

@@ -0,0 +1,163 @@
+package build.buildfarm.worker.util;
+
+import static build.buildfarm.worker.util.InputsIndexer.BAZEL_TOOL_INPUT_MARKER;
+import static com.google.common.truth.Truth.assertThat;
+
+import build.bazel.remote.execution.v2.*;
+import build.buildfarm.common.DigestUtil;
+import build.buildfarm.v1test.Tree;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.worker.WorkerProtocol.Input;
+import com.google.protobuf.ByteString;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.stream.Collectors;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class InputsIndexerTest {
+  private final DigestUtil DIGEST_UTIL = new DigestUtil(DigestUtil.HashFunction.SHA256);
+
+  @Test
+  public void basicEmptyTree() {
+    Tree emptyTree = Tree.newBuilder().build();
+    InputsIndexer indexer = new InputsIndexer(emptyTree);
+    assertThat(indexer.tree).isEqualTo(emptyTree);
+  }
+
+  @Test
+  public void canGetRootDir() {
+    Tree.Builder treeBuilder = Tree.newBuilder();
+
+    Directory rootDir = Directory.getDefaultInstance();
+    Digest rootDirDigest = addDirToTree(treeBuilder, "my_root_dir", rootDir);
+    treeBuilder.setRootDigest(rootDirDigest);
+
+    InputsIndexer indexer = new InputsIndexer(treeBuilder.build());
+    assertThat(indexer.proxyDirs.get(rootDirDigest)).isEqualTo(rootDir);
+
+    Path arbitraryOpRoot = Paths.get(".");
+    assertThat(indexer.getAllInputs(arbitraryOpRoot).size()).isEqualTo(0);
+  }
+
+  @Test
+  public void rootDirWithFiles() {
+    Tree.Builder treeBuilder = Tree.newBuilder();
+
+    FileNode myfile =
+        makeFileNode("my_file", "my file contents", NodeProperties.getDefaultInstance());
+    Directory rootDir = Directory.newBuilder().addFiles(myfile).build();
+    Digest rootDirDigest = addDirToTree(treeBuilder, "my_root_dir", rootDir);
+    treeBuilder.setRootDigest(rootDirDigest);
+
+    InputsIndexer indexer = new InputsIndexer(treeBuilder.build());
+    assertThat(indexer.proxyDirs.get(rootDirDigest)).isEqualTo(rootDir);
+
+    Path arbitraryOpRoot = Paths.get("asdf");
+    Input myfileInput = makeInput(arbitraryOpRoot, myfile);
+
+    ImmutableMap<Path, Input> expectedInputs =
+        ImmutableMap.of(Paths.get(myfileInput.getPath()), myfileInput);
+
+    assertThat(indexer.getAllInputs(arbitraryOpRoot)).isEqualTo(expectedInputs);
+  }
+
+  @Test
+  public void canRecurseAndDistinguishToolInputs() {
+    Tree.Builder treeBuilder = Tree.newBuilder();
+
+    FileNode myfile =
+        makeFileNode("my_file", "my file contents", NodeProperties.getDefaultInstance());
+    FileNode subdirfile =
+        makeFileNode("subdir_file", "my subdir file contents", NodeProperties.getDefaultInstance());
+    FileNode toolfile =
+        makeFileNode(
+            "tool_file",
+            "my tool file contents",
+            makeNodeProperties(ImmutableMap.of(BAZEL_TOOL_INPUT_MARKER, "value doesn't matter")));
+
+    Directory subDir = Directory.newBuilder().addFiles(subdirfile).build();
+    String subDirName = "my_sub_dir";
+    Digest subDirDigest = addDirToTree(treeBuilder, subDirName, subDir);
+
+    Directory rootDir =
+        Directory.newBuilder()
+            .addFiles(myfile)
+            .addFiles(toolfile)
+            .addDirectories(makeDirNode(subDirName, subDirDigest))
+            .build();
+
+    Digest rootDirDigest = addDirToTree(treeBuilder, "my_root_dir", rootDir);
+    treeBuilder.setRootDigest(rootDirDigest);
+
+    InputsIndexer indexer = new InputsIndexer(treeBuilder.build());
+    assertThat(indexer.proxyDirs.get(rootDirDigest)).isEqualTo(rootDir);
+    assertThat(indexer.proxyDirs.size()).isEqualTo(2);
+
+    Path arbitraryOpRoot = Paths.get("asdf");
+    Input myfileInput = makeInput(arbitraryOpRoot, myfile);
+    Input subdirfileInput = makeInput(arbitraryOpRoot.resolve(subDirName), subdirfile);
+    Input toolfileInput = makeInput(arbitraryOpRoot, toolfile);
+
+    ImmutableMap<Path, Input> nonToolInputs =
+        ImmutableMap.of(
+            Paths.get(myfileInput.getPath()),
+            myfileInput,
+            Paths.get(subdirfileInput.getPath()),
+            subdirfileInput);
+    ImmutableMap<Path, Input> toolInputs =
+        ImmutableMap.of(Paths.get(toolfileInput.getPath()), toolfileInput);
+    ImmutableMap<Path, Input> allInputs =
+        ImmutableMap.<Path, Input>builder().putAll(nonToolInputs).putAll(toolInputs).build();
+
+    assertThat(indexer.getAllInputs(arbitraryOpRoot)).isEqualTo(allInputs);
+    assertThat(indexer.getAllInputs(arbitraryOpRoot).size()).isEqualTo(3);
+    assertThat(indexer.getToolInputs(arbitraryOpRoot)).isEqualTo(toolInputs);
+  }
+
+  Digest addDirToTree(Tree.Builder treeBuilder, String dirname, Directory dir) {
+    ByteString dirnameBytes = ByteString.copyFromUtf8(dirname);
+    Digest digest = DIGEST_UTIL.compute(dirnameBytes);
+    String hash = digest.getHash();
+    treeBuilder.putDirectories(hash, dir);
+    return digest;
+  }
+
+  FileNode makeFileNode(String filename, String content, NodeProperties nodeProperties) {
+    return FileNode.newBuilder()
+        .setName(filename)
+        .setDigest(DIGEST_UTIL.compute(ByteString.copyFromUtf8(content)))
+        .setIsExecutable(false)
+        .setNodeProperties(nodeProperties)
+        .build();
+  }
+
+  DirectoryNode makeDirNode(String dirname, Digest dirDigest) {
+    // Pretty sure we don't need the actual hash for our testing purposes
+    return DirectoryNode.newBuilder().setName(dirname).setDigest(dirDigest).build();
+  }
+
+  NodeProperties makeNodeProperties(ImmutableMap<String, String> props) {
+    return NodeProperties.newBuilder()
+        .addAllProperties(
+            props.entrySet().stream()
+                .map(
+                    kv ->
+                        NodeProperty.newBuilder()
+                            .setName(kv.getKey())
+                            .setValue(kv.getValue())
+                            .build())
+                .collect(Collectors.toList()))
+        .build();
+  }
+
+  Input makeInput(Path fileDir, FileNode file) {
+    Path fileNodePath = fileDir.resolve(file.getName());
+    return Input.newBuilder()
+        .setPath(fileNodePath.toString())
+        .setDigest(file.getDigest().getHashBytes())
+        .build();
+  }
+}