forked from buildfarm/buildfarm
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Rename and add tests for util/InputIndexer and init PersistentExecuto…
…rTest
- Loading branch information
Showing
6 changed files
with
246 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| java_test( | ||
| name = "tests", | ||
| size = "small", | ||
| srcs = glob(["*.java"]), | ||
| test_class = "build.buildfarm.AllTests", | ||
| deps = [ | ||
| "//src/main/java/build/buildfarm/common", | ||
| "//src/main/java/build/buildfarm/common/config", | ||
| "//src/main/java/build/buildfarm/instance", | ||
| "//src/main/java/build/buildfarm/worker", | ||
| "//src/main/java/build/buildfarm/worker/resources", | ||
| "//src/main/protobuf:build_buildfarm_v1test_buildfarm_java_proto", | ||
| "//src/test/java/build/buildfarm:test_runner", | ||
| "@bazel_tools//src/main/protobuf:worker_protocol_java_proto", | ||
| "@googleapis//:google_rpc_code_java_proto", | ||
| "@maven//:com_github_jnr_jnr_constants", | ||
| "@maven//:com_github_jnr_jnr_ffi", | ||
| "@maven//:com_github_serceman_jnr_fuse", | ||
| "@maven//:com_google_guava_guava", | ||
| "@maven//:com_google_jimfs_jimfs", | ||
| "@maven//:com_google_protobuf_protobuf_java", | ||
| "@maven//:com_google_truth_truth", | ||
| "@maven//:io_grpc_grpc_api", | ||
| "@maven//:io_grpc_grpc_context", | ||
| "@maven//:io_grpc_grpc_core", | ||
| "@maven//:io_grpc_grpc_protobuf", | ||
| "@maven//:org_mockito_mockito_core", | ||
| "@maven//:org_projectlombok_lombok", | ||
| "@remote_apis//:build_bazel_remote_execution_v2_remote_execution_java_proto", | ||
| ], | ||
| ) |
13 changes: 13 additions & 0 deletions
13
src/test/java/build/buildfarm/worker/persistent/PersistentExecutorTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| package build.buildfarm.worker.persistent; | ||
|
|
||
| import org.junit.Test; | ||
| import org.junit.runner.RunWith; | ||
| import org.junit.runners.JUnit4; | ||
|
|
||
| @RunWith(JUnit4.class) | ||
| public class PersistentExecutorTest { | ||
| @Test | ||
| public void testProtoCoordinatorCreatesDirs() throws Exception { | ||
| assert (true); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| java_test( | ||
| name = "tests", | ||
| size = "small", | ||
| srcs = glob(["*.java"]), | ||
| test_class = "build.buildfarm.AllTests", | ||
| deps = [ | ||
| "//src/main/java/build/buildfarm/cas", | ||
| "//src/main/java/build/buildfarm/common", | ||
| "//src/main/java/build/buildfarm/common/config", | ||
| "//src/main/java/build/buildfarm/worker/util", | ||
| "//src/main/protobuf:build_buildfarm_v1test_buildfarm_java_proto", | ||
| "//src/test/java/build/buildfarm:test_runner", | ||
| "@bazel_tools//src/main/protobuf:worker_protocol_java_proto", | ||
| "@googleapis//:google_rpc_code_java_proto", | ||
| "@maven//:com_github_jnr_jnr_constants", | ||
| "@maven//:com_github_jnr_jnr_ffi", | ||
| "@maven//:com_github_serceman_jnr_fuse", | ||
| "@maven//:com_google_guava_guava", | ||
| "@maven//:com_google_jimfs_jimfs", | ||
| "@maven//:com_google_protobuf_protobuf_java", | ||
| "@maven//:com_google_truth_truth", | ||
| "@maven//:io_grpc_grpc_api", | ||
| "@maven//:io_grpc_grpc_context", | ||
| "@maven//:io_grpc_grpc_core", | ||
| "@maven//:io_grpc_grpc_protobuf", | ||
| "@maven//:org_mockito_mockito_core", | ||
| "@maven//:org_projectlombok_lombok", | ||
| "@remote_apis//:build_bazel_remote_execution_v2_remote_execution_java_proto", | ||
| ], | ||
| ) |
163 changes: 163 additions & 0 deletions
163
src/test/java/build/buildfarm/worker/util/InputsIndexerTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,163 @@ | ||
| package build.buildfarm.worker.util; | ||
|
|
||
| import static build.buildfarm.worker.util.InputsIndexer.BAZEL_TOOL_INPUT_MARKER; | ||
| import static com.google.common.truth.Truth.assertThat; | ||
|
|
||
| import build.bazel.remote.execution.v2.*; | ||
| import build.buildfarm.common.DigestUtil; | ||
| import build.buildfarm.v1test.Tree; | ||
| import com.google.common.collect.ImmutableMap; | ||
| import com.google.devtools.build.lib.worker.WorkerProtocol.Input; | ||
| import com.google.protobuf.ByteString; | ||
| import java.nio.file.Path; | ||
| import java.nio.file.Paths; | ||
| import java.util.stream.Collectors; | ||
| import org.junit.Test; | ||
| import org.junit.runner.RunWith; | ||
| import org.junit.runners.JUnit4; | ||
|
|
||
| @RunWith(JUnit4.class) | ||
| public class InputsIndexerTest { | ||
| private final DigestUtil DIGEST_UTIL = new DigestUtil(DigestUtil.HashFunction.SHA256); | ||
|
|
||
| @Test | ||
| public void basicEmptyTree() { | ||
| Tree emptyTree = Tree.newBuilder().build(); | ||
| InputsIndexer indexer = new InputsIndexer(emptyTree); | ||
| assertThat(indexer.tree).isEqualTo(emptyTree); | ||
| } | ||
|
|
||
| @Test | ||
| public void canGetRootDir() { | ||
| Tree.Builder treeBuilder = Tree.newBuilder(); | ||
|
|
||
| Directory rootDir = Directory.getDefaultInstance(); | ||
| Digest rootDirDigest = addDirToTree(treeBuilder, "my_root_dir", rootDir); | ||
| treeBuilder.setRootDigest(rootDirDigest); | ||
|
|
||
| InputsIndexer indexer = new InputsIndexer(treeBuilder.build()); | ||
| assertThat(indexer.proxyDirs.get(rootDirDigest)).isEqualTo(rootDir); | ||
|
|
||
| Path arbitraryOpRoot = Paths.get("."); | ||
| assertThat(indexer.getAllInputs(arbitraryOpRoot).size()).isEqualTo(0); | ||
| } | ||
|
|
||
| @Test | ||
| public void rootDirWithFiles() { | ||
| Tree.Builder treeBuilder = Tree.newBuilder(); | ||
|
|
||
| FileNode myfile = | ||
| makeFileNode("my_file", "my file contents", NodeProperties.getDefaultInstance()); | ||
| Directory rootDir = Directory.newBuilder().addFiles(myfile).build(); | ||
| Digest rootDirDigest = addDirToTree(treeBuilder, "my_root_dir", rootDir); | ||
| treeBuilder.setRootDigest(rootDirDigest); | ||
|
|
||
| InputsIndexer indexer = new InputsIndexer(treeBuilder.build()); | ||
| assertThat(indexer.proxyDirs.get(rootDirDigest)).isEqualTo(rootDir); | ||
|
|
||
| Path arbitraryOpRoot = Paths.get("asdf"); | ||
| Input myfileInput = makeInput(arbitraryOpRoot, myfile); | ||
|
|
||
| ImmutableMap<Path, Input> expectedInputs = | ||
| ImmutableMap.of(Paths.get(myfileInput.getPath()), myfileInput); | ||
|
|
||
| assertThat(indexer.getAllInputs(arbitraryOpRoot)).isEqualTo(expectedInputs); | ||
| } | ||
|
|
||
| @Test | ||
| public void canRecurseAndDistinguishToolInputs() { | ||
| Tree.Builder treeBuilder = Tree.newBuilder(); | ||
|
|
||
| FileNode myfile = | ||
| makeFileNode("my_file", "my file contents", NodeProperties.getDefaultInstance()); | ||
| FileNode subdirfile = | ||
| makeFileNode("subdir_file", "my subdir file contents", NodeProperties.getDefaultInstance()); | ||
| FileNode toolfile = | ||
| makeFileNode( | ||
| "tool_file", | ||
| "my tool file contents", | ||
| makeNodeProperties(ImmutableMap.of(BAZEL_TOOL_INPUT_MARKER, "value doesn't matter"))); | ||
|
|
||
| Directory subDir = Directory.newBuilder().addFiles(subdirfile).build(); | ||
| String subDirName = "my_sub_dir"; | ||
| Digest subDirDigest = addDirToTree(treeBuilder, subDirName, subDir); | ||
|
|
||
| Directory rootDir = | ||
| Directory.newBuilder() | ||
| .addFiles(myfile) | ||
| .addFiles(toolfile) | ||
| .addDirectories(makeDirNode(subDirName, subDirDigest)) | ||
| .build(); | ||
|
|
||
| Digest rootDirDigest = addDirToTree(treeBuilder, "my_root_dir", rootDir); | ||
| treeBuilder.setRootDigest(rootDirDigest); | ||
|
|
||
| InputsIndexer indexer = new InputsIndexer(treeBuilder.build()); | ||
| assertThat(indexer.proxyDirs.get(rootDirDigest)).isEqualTo(rootDir); | ||
| assertThat(indexer.proxyDirs.size()).isEqualTo(2); | ||
|
|
||
| Path arbitraryOpRoot = Paths.get("asdf"); | ||
| Input myfileInput = makeInput(arbitraryOpRoot, myfile); | ||
| Input subdirfileInput = makeInput(arbitraryOpRoot.resolve(subDirName), subdirfile); | ||
| Input toolfileInput = makeInput(arbitraryOpRoot, toolfile); | ||
|
|
||
| ImmutableMap<Path, Input> nonToolInputs = | ||
| ImmutableMap.of( | ||
| Paths.get(myfileInput.getPath()), | ||
| myfileInput, | ||
| Paths.get(subdirfileInput.getPath()), | ||
| subdirfileInput); | ||
| ImmutableMap<Path, Input> toolInputs = | ||
| ImmutableMap.of(Paths.get(toolfileInput.getPath()), toolfileInput); | ||
| ImmutableMap<Path, Input> allInputs = | ||
| ImmutableMap.<Path, Input>builder().putAll(nonToolInputs).putAll(toolInputs).build(); | ||
|
|
||
| assertThat(indexer.getAllInputs(arbitraryOpRoot)).isEqualTo(allInputs); | ||
| assertThat(indexer.getAllInputs(arbitraryOpRoot).size()).isEqualTo(3); | ||
| assertThat(indexer.getToolInputs(arbitraryOpRoot)).isEqualTo(toolInputs); | ||
| } | ||
|
|
||
| Digest addDirToTree(Tree.Builder treeBuilder, String dirname, Directory dir) { | ||
| ByteString dirnameBytes = ByteString.copyFromUtf8(dirname); | ||
| Digest digest = DIGEST_UTIL.compute(dirnameBytes); | ||
| String hash = digest.getHash(); | ||
| treeBuilder.putDirectories(hash, dir); | ||
| return digest; | ||
| } | ||
|
|
||
| FileNode makeFileNode(String filename, String content, NodeProperties nodeProperties) { | ||
| return FileNode.newBuilder() | ||
| .setName(filename) | ||
| .setDigest(DIGEST_UTIL.compute(ByteString.copyFromUtf8(content))) | ||
| .setIsExecutable(false) | ||
| .setNodeProperties(nodeProperties) | ||
| .build(); | ||
| } | ||
|
|
||
| DirectoryNode makeDirNode(String dirname, Digest dirDigest) { | ||
| // Pretty sure we don't need the actual hash for our testing purposes | ||
| return DirectoryNode.newBuilder().setName(dirname).setDigest(dirDigest).build(); | ||
| } | ||
|
|
||
| NodeProperties makeNodeProperties(ImmutableMap<String, String> props) { | ||
| return NodeProperties.newBuilder() | ||
| .addAllProperties( | ||
| props.entrySet().stream() | ||
| .map( | ||
| kv -> | ||
| NodeProperty.newBuilder() | ||
| .setName(kv.getKey()) | ||
| .setValue(kv.getValue()) | ||
| .build()) | ||
| .collect(Collectors.toList())) | ||
| .build(); | ||
| } | ||
|
|
||
| Input makeInput(Path fileDir, FileNode file) { | ||
| Path fileNodePath = fileDir.resolve(file.getName()); | ||
| return Input.newBuilder() | ||
| .setPath(fileNodePath.toString()) | ||
| .setDigest(file.getDigest().getHashBytes()) | ||
| .build(); | ||
| } | ||
| } |