Skip to content

Commit

Permalink
WIP: Make QuarkusBuild not pollute Gradle's build cache
Browse files Browse the repository at this point in the history
TODO: find a way to allow caching "large" uber-jars and native runnables (for those who want that)

* Removes the huge `lib/` folder from the output-directory of the `QuarkusBuild` task to reduce the size of the cache entity in Gradle's build cache.
* Introduce a new `QuarkusBuildLibs` task to produce the `lib/` folder and its content. This task's "up-to-date" checks works, but it is intentionally not cacheable (it's huge). Re-creating the contents of the `lib/` folder is rather cheap and the dependency artifacts are mangaged by the local artifact repository or via the project's build artifacts (jars generated by other modules of the same build).
* Introduce a new `QuarkusBuildFinish` task to combine the output of the "`lib/`-less" `QuarkusBuild` and the output of the `QuarkusBuildLibs` tasks, so the result is the same as before this change.

Other, related changes:
* The `QuarkusBuild` task intentionally removes outputs for other package types than the currently configured one. This makes up-to-date checks work across multiple package types.

Other notes:
* The task names `quarkusLibsBuild` and `quarkusFinishBuild` are intentionally "that way around". Letting the names of these tasks begin with `quarkusBuild...` could confuse users, who use abbreviated task names on the command line (for example `./gradlew qB` is automagically expanded to `./gradlew quarkusBuild`, which would become ambiguous with `quarkusBuildLibs` and `quarkusBuildFinish`).

Relates to: quarkusio#30852
  • Loading branch information
snazy committed Feb 4, 2023
1 parent aec9d78 commit 3a8ebdc
Show file tree
Hide file tree
Showing 7 changed files with 293 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@
import io.quarkus.gradle.tasks.QuarkusAddExtension;
import io.quarkus.gradle.tasks.QuarkusBuild;
import io.quarkus.gradle.tasks.QuarkusBuildConfiguration;
import io.quarkus.gradle.tasks.QuarkusBuildFinish;
import io.quarkus.gradle.tasks.QuarkusBuildLibs;
import io.quarkus.gradle.tasks.QuarkusDev;
import io.quarkus.gradle.tasks.QuarkusGenerateCode;
import io.quarkus.gradle.tasks.QuarkusGoOffline;
Expand Down Expand Up @@ -66,6 +68,8 @@ public class QuarkusPlugin implements Plugin<Project> {
public static final String QUARKUS_GENERATE_CODE_TASK_NAME = "quarkusGenerateCode";
public static final String QUARKUS_GENERATE_CODE_DEV_TASK_NAME = "quarkusGenerateCodeDev";
public static final String QUARKUS_GENERATE_CODE_TESTS_TASK_NAME = "quarkusGenerateCodeTests";
public static final String QUARKUS_BUILD_LIBS_TASK_NAME = "quarkusLibsBuild";
public static final String QUARKUS_BUILD_FINISH_TASK_NAME = "quarkusFinishBuild";
public static final String QUARKUS_BUILD_TASK_NAME = "quarkusBuild";
public static final String QUARKUS_DEV_TASK_NAME = "quarkusDev";
public static final String QUARKUS_REMOTE_DEV_TASK_NAME = "quarkusRemoteDev";
Expand Down Expand Up @@ -150,9 +154,27 @@ private void registerTasks(Project project, QuarkusPluginExtension quarkusExt) {

QuarkusBuildConfiguration buildConfig = new QuarkusBuildConfiguration(project);

TaskProvider<QuarkusBuildLibs> quarkusBuildLibs = tasks.register(QUARKUS_BUILD_LIBS_TASK_NAME,
QuarkusBuildLibs.class);

TaskProvider<QuarkusBuildFinish> quarkusBuildFinalize = tasks.register(QUARKUS_BUILD_FINISH_TASK_NAME,
QuarkusBuildFinish.class);

TaskProvider<QuarkusBuild> quarkusBuild = tasks.register(QUARKUS_BUILD_TASK_NAME, QuarkusBuild.class, build -> {
build.dependsOn(quarkusGenerateCode);
build.finalizedBy(quarkusBuildFinalize);
build.getForcedProperties().set(buildConfig.getForcedProperties());
build.getOutputs().doNotCacheIf("Caching disabled for configured package type",
t -> !((QuarkusBuild) t).isFastJarLike());
});

quarkusBuildLibs.configure(task -> {
task.getOutputs().doNotCacheIf("Collecting dependencies not worth to cache", t -> true);
});

quarkusBuildFinalize.configure(task -> {
task.dependsOn(quarkusBuildLibs, quarkusBuild);
task.getOutputs().doNotCacheIf("Assembling task not worth to cache", t -> true);
});

TaskProvider<ImageBuild> imageBuild = tasks.register(IMAGE_BUILD_TASK_NAME, ImageBuild.class, buildConfig);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.EnumMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Objects;
Expand Down Expand Up @@ -164,9 +165,12 @@ public ApplicationModel getApplicationModel() {
}

public ApplicationModel getApplicationModel(LaunchMode mode) {
return ToolingUtils.create(project, mode);
// Prevent duplicate computation of ApplicationModel(s), same model's needed by multiple tasks.
return applicationModels.computeIfAbsent(mode, m -> ToolingUtils.create(project, m));
}

private transient Map<LaunchMode, ApplicationModel> applicationModels = new EnumMap<>(LaunchMode.class);

/**
* Returns the last file from the specified {@link FileCollection}.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import java.io.FileInputStream;
import java.io.IOException;
import java.io.Serializable;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
Expand All @@ -16,6 +17,7 @@
import org.gradle.api.Action;
import org.gradle.api.GradleException;
import org.gradle.api.file.FileCollection;
import org.gradle.api.file.FileSystemOperations;
import org.gradle.api.java.archives.Attributes;
import org.gradle.api.provider.MapProperty;
import org.gradle.api.tasks.CacheableTask;
Expand All @@ -34,6 +36,7 @@
import io.quarkus.bootstrap.app.QuarkusBootstrap;
import io.quarkus.bootstrap.model.ApplicationModel;
import io.quarkus.bootstrap.resolver.AppModelResolverException;
import io.quarkus.gradle.QuarkusPlugin;
import io.quarkus.gradle.dsl.Manifest;
import io.quarkus.maven.dependency.GACTV;
import io.quarkus.runtime.util.StringUtil;
Expand All @@ -46,6 +49,7 @@ public abstract class QuarkusBuild extends QuarkusTask {
private static final String MANIFEST_ATTRIBUTES_PROPERTY_PREFIX = "quarkus.package.manifest.attributes";

private static final String OUTPUT_DIRECTORY = "quarkus.package.output-directory";
private static final String QUARKUS_BUILD_DIR = "quarkus-build-dir";

private List<String> ignoredEntries = new ArrayList<>();
private Manifest manifest = new Manifest();
Expand All @@ -70,6 +74,9 @@ public QuarkusBuild nativeArgs(Action<Map<String, ?>> action) {
return this;
}

@Inject
public abstract FileSystemOperations getFileSystemOperations();

@Optional
@Input
public abstract MapProperty<String, String> getForcedProperties();
Expand Down Expand Up @@ -122,6 +129,22 @@ public Manifest getManifest() {
return this.manifest;
}

@Input
public String getPackageType() {
return getPropValueWithPrecedence(QuarkusPlugin.QUARKUS_PACKAGE_TYPE, java.util.Optional.of("fast-jar"));
}

@Internal
public boolean isFastJarLike() {
switch (getPackageType()) {
case "fast-jar":
case "native":
return true;
default:
return false;
}
}

public QuarkusBuild manifest(Action<Manifest> action) {
action.execute(this.getManifest());
return this;
Expand All @@ -137,17 +160,42 @@ public File getNativeRunner() {
return new File(getProject().getBuildDir(), extension().buildNativeRunnerName(Map.of()));
}

@OutputDirectory
@Internal
public File getFastJar() {
return new File(getProject().getBuildDir(),
this.getPropValueWithPrecedence(OUTPUT_DIRECTORY, java.util.Optional.of("quarkus-app")));
return new File(getProject().getBuildDir(), getOutputDirectory());
}

/**
* Contains the Quarkus app, but without the {@code lib/} folder. Contents of this directory are not too big and
* should be cacheable.
*/
@OutputDirectory
public File getFastJarBuildDir() {
return new File(getProject().getBuildDir(), QUARKUS_BUILD_DIR);
}

/**
* Directory into which the {@link QuarkusBuildLibs} populates the dependencies.
*/
@Internal
public File getLibsBuildDir() {
return new File(getProject().getBuildDir(), "quarkus-build-libs");
}

@Internal
public String getOutputDirectory() {
return getPropValueWithPrecedence(OUTPUT_DIRECTORY, java.util.Optional.of("quarkus-app"));
}

@TaskAction
public void buildQuarkus() {
final ApplicationModel appModel;
final Map<String, String> forcedProperties = getForcedProperties().getOrElse(Collections.emptyMap());

// Caching and "up-to-date" checks depend on the inputs, this 'delete()' should ensure that the up-to-date
// checks work against "clean" outputs, considering that the outputs depend on the package-type.
getFileSystemOperations().delete(delete -> delete.delete(getRunnerJar(), getFastJar(), getFastJarBuildDir()));

try {
appModel = extension().getAppModelResolver().resolveModel(new GACTV(getProject().getGroup().toString(),
getProject().getName(), getProject().getVersion().toString()));
Expand All @@ -164,10 +212,15 @@ public void buildQuarkus() {

exportCustomManifestProperties(effectiveProperties);

boolean fastJarLike = isFastJarLike();
Path quarkusBuildDir = (fastJarLike ? getFastJarBuildDir() : getProject().getBuildDir()).toPath();

getLogger().info("Building in target directory {}", quarkusBuildDir);

try (CuratedApplication appCreationContext = QuarkusBootstrap.builder()
.setBaseClassLoader(getClass().getClassLoader())
.setExistingModel(appModel)
.setTargetDirectory(getProject().getBuildDir().toPath())
.setTargetDirectory(quarkusBuildDir)
.setBaseName(extension().finalName())
.setBuildSystemProperties(effectiveProperties)
.setAppArtifact(appModel.getAppArtifact())
Expand All @@ -184,6 +237,10 @@ public void buildQuarkus() {
appCreationContext.createAugmentor("io.quarkus.deployment.pkg.builditem.ProcessInheritIODisabled$Factory",
Collections.emptyMap()).createProductionApplication();

if (fastJarLike) {
Path resolvedAppBuildDir = getFastJarBuildDir().toPath().resolve(getOutputDirectory());
getFileSystemOperations().delete(delete -> delete.delete(resolvedAppBuildDir.resolve("lib")));
}
} catch (BootstrapException e) {
throw new GradleException("Failed to build a runnable JAR", e);
}
Expand Down Expand Up @@ -233,7 +290,7 @@ private String expandConfigurationKey(String shortKey) {
return String.format("%s.%s", NATIVE_PROPERTY_NAMESPACE, hyphenatedKey);
}

private String getPropValueWithPrecedence(final String propName, final java.util.Optional<String> defaultValue) {
String getPropValueWithPrecedence(final String propName, final java.util.Optional<String> defaultValue) {
if (applicationProperties.isEmpty()) {
SourceSet mainSourceSet = QuarkusGradleUtils.getSourceSet(getProject(), SourceSet.MAIN_SOURCE_SET_NAME);

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package io.quarkus.gradle.tasks;

import java.io.File;

import javax.inject.Inject;

import org.gradle.api.file.FileSystemOperations;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.OutputDirectory;
import org.gradle.api.tasks.OutputFile;
import org.gradle.api.tasks.TaskAction;

/**
* Finalizes the build of a Quarkus app, combining the outputs of {@link QuarkusBuild} and {@link QuarkusBuildLibs}.
*
* <p>
* This task is required to "properly" cache the output of {@link QuarkusBuild}
*/
public abstract class QuarkusBuildFinish extends QuarkusTask {

public static final String QUARKUS_ARTIFACT_PROPERTIES = "quarkus-artifact.properties";

@Inject
public QuarkusBuildFinish() {
super("Finalize the Quarkus application build, prefer the 'quarkusBuild' task");
}

@Inject
public abstract FileSystemOperations getFileSystemOperations();

@Input
public String getPackageType() {
return quarkusBuild().getPackageType();
}

@OutputDirectory
public File getFastJar() {
return quarkusBuild().getFastJar();
}

@OutputFile
public File getArtifactPropertiesFile() {
return new File(getFastJar().getParentFile(), QUARKUS_ARTIFACT_PROPERTIES);
}

@TaskAction
public void run() {
File finalDir = getFastJar();

QuarkusBuild quarkusBuild = quarkusBuild();
if (!quarkusBuild.isFastJarLike()) {
getLogger().info("Nothing to do for package type {}", quarkusBuild.getPackageType());
getFileSystemOperations().delete(delete -> delete.delete(finalDir));
return;
}

String outputDir = quarkusBuild.getOutputDirectory();
File appBuildDir = new File(quarkusBuild.getFastJarBuildDir(), outputDir);
File libsBuildDir = new File(quarkusBuild.getLibsBuildDir(), outputDir);

getLogger().info("Finalizing Quarkus build in {} from {} and {}", finalDir, appBuildDir, libsBuildDir);

getFileSystemOperations().sync(sync -> {
sync.into(finalDir);
sync.from(appBuildDir, libsBuildDir);
});

getFileSystemOperations().copy(
copy -> copy.into(finalDir.getParent()).from(appBuildDir.getParent()).include(QUARKUS_ARTIFACT_PROPERTIES));
}
}
Loading

0 comments on commit 3a8ebdc

Please sign in to comment.