Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[7.1.0] Introduce a SpawnLogContext interface. #20842

Merged
merged 1 commit into from
Jan 10, 2024
Merged

[7.1.0] Introduce a SpawnLogContext interface. #20842

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions src/main/java/com/google/devtools/build/lib/bazel/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,6 @@ java_library(
],
deps = [
"//src/main/java/com/google/devtools/build/lib:runtime",
"//src/main/java/com/google/devtools/build/lib/bazel/execlog:stable_sort",
"//src/main/java/com/google/devtools/build/lib/events",
"//src/main/java/com/google/devtools/build/lib/exec:execution_options",
"//src/main/java/com/google/devtools/build/lib/exec:executor_builder",
Expand All @@ -138,10 +137,8 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/remote/options",
"//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception",
"//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code",
"//src/main/java/com/google/devtools/build/lib/util/io",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/protobuf:failure_details_java_proto",
"//src/main/protobuf:spawn_java_proto",
"//third_party:guava",
"//third_party:jsr305",
],
Expand Down
107 changes: 18 additions & 89 deletions src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,13 @@

import static com.google.common.base.Preconditions.checkNotNull;

import com.google.devtools.build.lib.bazel.execlog.StableSort;
import com.google.devtools.build.lib.buildtool.BuildRequest;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.exec.ExecutionOptions;
import com.google.devtools.build.lib.exec.ExecutorBuilder;
import com.google.devtools.build.lib.exec.ExpandedSpawnLogContext;
import com.google.devtools.build.lib.exec.ExpandedSpawnLogContext.Encoding;
import com.google.devtools.build.lib.exec.ModuleActionContextRegistry;
import com.google.devtools.build.lib.exec.Protos.SpawnExec;
import com.google.devtools.build.lib.exec.SpawnLogContext;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
import com.google.devtools.build.lib.runtime.BlazeModule;
Expand All @@ -32,41 +32,16 @@
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.util.AbruptExitException;
import com.google.devtools.build.lib.util.DetailedExitCode;
import com.google.devtools.build.lib.util.io.AsynchronousMessageOutputStream;
import com.google.devtools.build.lib.util.io.MessageOutputStream;
import com.google.devtools.build.lib.util.io.MessageOutputStreamWrapper.BinaryOutputStreamWrapper;
import com.google.devtools.build.lib.util.io.MessageOutputStreamWrapper.JsonOutputStreamWrapper;
import com.google.devtools.build.lib.vfs.Path;
import java.io.IOException;
import java.io.InputStream;
import javax.annotation.Nullable;

/** Module providing on-demand spawn logging. */
public final class SpawnLogModule extends BlazeModule {
@Nullable private SpawnLogContext spawnLogContext;

/** Output path for the raw output stream. */
@Nullable private Path rawOutputPath;

/** Output stream to write directly into during execution. */
@Nullable private MessageOutputStream<SpawnExec> rawOutputStream;

/**
* Output stream to convert the raw output into after the execution is done.
*
* <p>We open the stream at the beginning of the command so that any errors (e.g., unwritable
* location) are surfaced before execution begins.
*/
@Nullable private MessageOutputStream<SpawnExec> convertedOutputStream;

private CommandEnvironment env;

private void clear() {
spawnLogContext = null;
rawOutputPath = null;
rawOutputStream = null;
convertedOutputStream = null;
env = null;
}

private void initOutputs(CommandEnvironment env) throws IOException {
Expand Down Expand Up @@ -99,45 +74,30 @@ private void initOutputs(CommandEnvironment env) throws IOException {
return;
}

this.env = env;

Path workingDirectory = env.getWorkingDirectory();
Path outputBase = env.getOutputBase();

// Set up the raw output stream.
// This stream performs the writes in a separate thread to avoid blocking execution.
// If the unsorted binary format was requested, use the respective output path to avoid a
// pointless conversion at the end. Otherwise, use a temporary path.
if (executionOptions.executionLogBinaryFile != null && !executionOptions.executionLogSort) {
rawOutputPath = workingDirectory.getRelative(executionOptions.executionLogBinaryFile);
} else {
rawOutputPath = outputBase.getRelative("execution.log");
}
rawOutputStream = new AsynchronousMessageOutputStream<>(rawOutputPath);

// Set up the binary output stream, if distinct from the raw output stream.
if (executionOptions.executionLogBinaryFile != null && executionOptions.executionLogSort) {
convertedOutputStream =
new BinaryOutputStreamWrapper<>(
workingDirectory
.getRelative(executionOptions.executionLogBinaryFile)
.getOutputStream());
Path outputPath = null;
Encoding encoding = null;
if (executionOptions.executionLogBinaryFile != null) {
encoding = Encoding.BINARY;
outputPath = workingDirectory.getRelative(executionOptions.executionLogBinaryFile);
} else if (executionOptions.executionLogJsonFile != null) {
encoding = Encoding.JSON;
outputPath = workingDirectory.getRelative(executionOptions.executionLogJsonFile);
}

// Set up the text output stream.
if (executionOptions.executionLogJsonFile != null) {
convertedOutputStream =
new JsonOutputStreamWrapper<>(
workingDirectory
.getRelative(executionOptions.executionLogJsonFile)
.getOutputStream());
}
// Use a well-known temporary path to avoid accumulation of potentially large files in /tmp
// due to abnormally terminated invocations (e.g., when running out of memory).
Path tempPath = outputBase.getRelative("execution.log");

spawnLogContext =
new SpawnLogContext(
new ExpandedSpawnLogContext(
checkNotNull(outputPath),
tempPath,
checkNotNull(encoding),
/* sorted= */ executionOptions.executionLogSort,
env.getExecRoot().asFragment(),
rawOutputStream,
env.getOptions().getOptions(ExecutionOptions.class),
env.getOptions().getOptions(RemoteOptions.class),
env.getRuntime().getFileSystem().getDigestFunction(),
env.getXattrProvider());
Expand Down Expand Up @@ -180,44 +140,13 @@ public void afterCommand() throws AbruptExitException {
return;
}

checkNotNull(rawOutputPath);

boolean done = false;
try {
spawnLogContext.close();
if (convertedOutputStream != null) {
InputStream in = rawOutputPath.getInputStream();
if (spawnLogContext.shouldSort()) {
StableSort.stableSort(in, convertedOutputStream);
} else {
while (in.available() > 0) {
SpawnExec ex = SpawnExec.parseDelimitedFrom(in);
convertedOutputStream.write(ex);
}
}
convertedOutputStream.close();
}
done = true;
} catch (IOException e) {
String message = e.getMessage() == null ? "Error writing execution log" : e.getMessage();
throw new AbruptExitException(
createDetailedExitCode(message, Code.EXECUTION_LOG_WRITE_FAILURE), e);
} finally {
if (convertedOutputStream != null) {
if (!done) {
env.getReporter()
.handle(
Event.warn(
"Execution log might not have been populated. Raw execution log is at "
+ rawOutputPath));
} else {
try {
rawOutputPath.delete();
} catch (IOException e) {
// Intentionally ignored.
}
}
}
clear();
}
}
Expand Down
7 changes: 5 additions & 2 deletions src/main/java/com/google/devtools/build/lib/exec/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,13 +257,16 @@ java_library(

java_library(
name = "spawn_log_context",
srcs = ["SpawnLogContext.java"],
srcs = [
"ExpandedSpawnLogContext.java",
"SpawnLogContext.java",
],
deps = [
":execution_options",
"//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/actions:artifacts",
"//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
"//src/main/java/com/google/devtools/build/lib/analysis/platform:platform_utils",
"//src/main/java/com/google/devtools/build/lib/bazel/execlog:stable_sort",
"//src/main/java/com/google/devtools/build/lib/profiler",
"//src/main/java/com/google/devtools/build/lib/remote/options",
"//src/main/java/com/google/devtools/build/lib/util/io",
Expand Down
Loading
Loading