From c092355bd16c0a34c70e0efb45f87077fb1b7eea Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Sat, 4 Feb 2023 23:04:35 +0100 Subject: [PATCH] WIP: Make `QuarkusBuild` not pollute Gradle's build cache Currently the `QuarkusBuild` task implementation adds even large build artifacts and unmodified dependency jars to Gradle's build cache. This pollutes the Gradle build cache quite a lot and therefore lets archived build caches become unnecessary huge. This change updates the build logic to fix this behavior by not adding dependencies and large build artifacts, like uber-jar and native binary, to the Gradle build cache. The `QuarkusBuild` task has been split into three tasks: 1. a new `QuarkusBuildDependencies` task that only collects the contents for `build/quarkus-app/lib` 2. a new `QuarkusBuildApp` task that to collect everything else from a Quarkus build (everything except the `build/quarkus-app/lib`) 3. the `QuarkusBuild` task now combines the outputs of the above two tasks `QuarkusBuildDependencies` (named 'quarkusDependenciesBuild`) is not cacheable, because it only collects dependencies, which come either from a repository (and are already available locally elsewhere) or are built by other Gradle tasks. This task is only executed if the configured Quarkus package type requires the "quarkus-app" directory (`fast-jar` + `native`). It's "build working directory" is `build/quarkus-build/dep`. `QuarkusBuildApp` (named `quarkusAppBuild`) collects the contents of the "quarkus-app" directory _excluding_ the `lib/` directory are cacheable, which is the default for CI environments. Non-CI environments still cache all outputs, even uber-jars and native binaries to retain the existing behavior for developers and keep build turn-around times low. CI environments can opt-in to add even huge artifacts to Gradle's build cache by explicitly setting the `cacheUberAndNativeRunners` property in the Quarkus extension to `true`. It's "build working directory" is `build/quarkus-build/app`. Since `QuarkusBuild` only combines the outputs of the above two tasks, the same "CI vs local" caching behavior as for the `QuarkusBuildApp` task applies. To make "up to date" checks (kind of) more reliable, all outputs are removed first. This means, that for example an existing uber-jar in `build/` will disappear, when the build's package type is "fast-jar". This behavior can be disabled by setting the `cleanupBuildOutput` property on the Quarkus extension to `false`. Both `QuarkusBuildDependencies` and `QuarkusBuildApp` can trigger an actual Quarkus application build. That Quarkus app build will only be triggered when needed and only once per Gradle build. The task names `quarkusDependenciesBuild` and `quarkusAppBuild` are intentionally "that way around". Letting the names of these tasks begin with `quarkusBuild...` could confuse users, who use abbreviated task names on the command line (for example `./gradlew qB` is automagically expanded to `./gradlew quarkusBuild`, which would become ambiguous with `quarkusBuildDependencies` and `quarkusBuildApp`). Unless the `cacheLargeArtifacts` property on the `quarkus` extension is set to `true`, the output of/for the package type `fast-jar` minus the dependency jars is cached by Gradle's build cache (similar for `legacy-jar`). Basically everything is cacheable to allow fast(er) local development turn-around cycles. Relates to: #30852 --- .../gradle-application-plugin/build.gradle | 1 + .../gradle/gradle-application-plugin/pom.xml | 4 + .../java/io/quarkus/gradle/QuarkusPlugin.java | 46 ++- .../extension/QuarkusPluginExtension.java | 85 +++- .../io/quarkus/gradle/tasks/QuarkusBuild.java | 318 ++++++--------- .../quarkus/gradle/tasks/QuarkusBuildApp.java | 202 ++++++++++ .../tasks/QuarkusBuildConfiguration.java | 378 +++++++++++++++++- .../tasks/QuarkusBuildDependencies.java | 162 ++++++++ .../gradle/tasks/QuarkusBuildTask.java | 90 +++++ .../gradle/tasks/QuarkusGenerateCode.java | 5 + devtools/gradle/gradle.properties | 1 + .../nativeimage/NativeIntegrationTestIT.java | 6 +- 12 files changed, 1062 insertions(+), 236 deletions(-) create mode 100644 devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuildApp.java create mode 100644 devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuildDependencies.java create mode 100644 devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuildTask.java diff --git a/devtools/gradle/gradle-application-plugin/build.gradle b/devtools/gradle/gradle-application-plugin/build.gradle index 57cd56b24de053..525e3b118852f2 100644 --- a/devtools/gradle/gradle-application-plugin/build.gradle +++ b/devtools/gradle/gradle-application-plugin/build.gradle @@ -7,6 +7,7 @@ dependencies { implementation "io.quarkus:quarkus-devtools-common:${version}" implementation "io.quarkus:quarkus-core-deployment:${version}" implementation "io.quarkus:quarkus-bootstrap-gradle-resolver:${version}" + implementation "io.smallrye.config:smallrye-config-source-yaml:${smallrye_config_version}" implementation project(":gradle-model") diff --git a/devtools/gradle/gradle-application-plugin/pom.xml b/devtools/gradle/gradle-application-plugin/pom.xml index 49a6900cfb3881..8ec8343394d8e1 100644 --- a/devtools/gradle/gradle-application-plugin/pom.xml +++ b/devtools/gradle/gradle-application-plugin/pom.xml @@ -54,6 +54,10 @@ quarkus-devmode-test-utils test + + io.smallrye.config + smallrye-config-source-yaml + org.jetbrains.kotlin kotlin-gradle-plugin diff --git a/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/QuarkusPlugin.java b/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/QuarkusPlugin.java index aa33e610677870..1d3ffc8787ba7d 100644 --- a/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/QuarkusPlugin.java +++ b/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/QuarkusPlugin.java @@ -36,7 +36,9 @@ import io.quarkus.gradle.tasks.ImagePush; import io.quarkus.gradle.tasks.QuarkusAddExtension; import io.quarkus.gradle.tasks.QuarkusBuild; +import io.quarkus.gradle.tasks.QuarkusBuildApp; import io.quarkus.gradle.tasks.QuarkusBuildConfiguration; +import io.quarkus.gradle.tasks.QuarkusBuildDependencies; import io.quarkus.gradle.tasks.QuarkusDev; import io.quarkus.gradle.tasks.QuarkusGenerateCode; import io.quarkus.gradle.tasks.QuarkusGoOffline; @@ -55,7 +57,17 @@ public class QuarkusPlugin implements Plugin { public static final String ID = "io.quarkus"; + public static final String QUARKUS_PROFILE = "quarkus.profile"; + public static final String DEFAULT_PROFILE = "prod"; + public static final String QUARKUS_PACKAGE_OUTPUT_NAME = "quarkus.package.output-name"; + public static final String QUARKUS_PACKAGE_ADD_RUNNER_SUFFIX = "quarkus.package.add-runner-suffix"; public static final String QUARKUS_PACKAGE_TYPE = "quarkus.package.type"; + public static final String DEFAULT_PACKAGE_TYPE = "jar"; + public static final String OUTPUT_DIRECTORY = "quarkus.package.output-directory"; + public static final String DEFAULT_OUTPUT_DIRECTORY = "quarkus-app"; + public static final String CLASS_LOADING_REMOVED_ARTIFACTS = "quarkus.class-loading.removed-artifacts"; + public static final String CLASS_LOADING_PARENT_FIRST_ARTIFACTS = "quarkus.class-loading.parent-first-artifacts"; + public static final String QUARKUS_ARTIFACT_PROPERTIES = "quarkus-artifact.properties"; public static final String EXTENSION_NAME = "quarkus"; public static final String LIST_EXTENSIONS_TASK_NAME = "listExtensions"; @@ -66,6 +78,8 @@ public class QuarkusPlugin implements Plugin { public static final String QUARKUS_GENERATE_CODE_TASK_NAME = "quarkusGenerateCode"; public static final String QUARKUS_GENERATE_CODE_DEV_TASK_NAME = "quarkusGenerateCodeDev"; public static final String QUARKUS_GENERATE_CODE_TESTS_TASK_NAME = "quarkusGenerateCodeTests"; + public static final String QUARKUS_BUILD_DEP_TASK_NAME = "quarkusDependenciesBuild"; + public static final String QUARKUS_BUILD_APP_TASK_NAME = "quarkusAppBuild"; public static final String QUARKUS_BUILD_TASK_NAME = "quarkusBuild"; public static final String QUARKUS_DEV_TASK_NAME = "quarkusDev"; public static final String QUARKUS_REMOTE_DEV_TASK_NAME = "quarkusRemoteDev"; @@ -96,6 +110,7 @@ public class QuarkusPlugin implements Plugin { private final ToolingModelBuilderRegistry registry; + @SuppressWarnings("CdiInjectionPointsInspection") @Inject public QuarkusPlugin(ToolingModelBuilderRegistry registry) { this.registry = registry; @@ -148,11 +163,30 @@ private void registerTasks(Project project, QuarkusPluginExtension quarkusExt) { TaskProvider quarkusGenerateCodeTests = tasks.register(QUARKUS_GENERATE_CODE_TESTS_TASK_NAME, QuarkusGenerateCode.class, task -> task.setTest(true)); - QuarkusBuildConfiguration buildConfig = new QuarkusBuildConfiguration(project); + QuarkusBuildConfiguration buildConfig = new QuarkusBuildConfiguration(project, quarkusExt); - TaskProvider quarkusBuild = tasks.register(QUARKUS_BUILD_TASK_NAME, QuarkusBuild.class, build -> { - build.dependsOn(quarkusGenerateCode); - build.getForcedProperties().set(buildConfig.getForcedProperties()); + TaskProvider quarkusBuildDep = tasks.register(QUARKUS_BUILD_DEP_TASK_NAME, + QuarkusBuildDependencies.class, buildConfig); + quarkusBuildDep.configure(task -> { + task.getOutputs().doNotCacheIf("Dependencies are never cached", t -> true); + }); + + TaskProvider quarkusBuildApp = tasks.register(QUARKUS_BUILD_APP_TASK_NAME, + QuarkusBuildApp.class, buildConfig); + quarkusBuildApp.configure(task -> { + task.dependsOn(quarkusGenerateCode); + task.getOutputs().doNotCacheIf( + "Not adding uber-jars and native binaries to Gradle build cache by default. " + + "To allow caching of uber-jars and native binaries set 'cacheUberAndNativeRunners' to 'true' in the 'quarkus' extension.", + t -> !quarkusExt.getCacheLargeArtifacts().get()); + }); + + TaskProvider quarkusBuild = tasks.register(QUARKUS_BUILD_TASK_NAME, QuarkusBuild.class, buildConfig); + quarkusBuild.configure(build -> { + build.dependsOn(quarkusGenerateCode, quarkusBuildDep, quarkusBuildApp); + build.getOutputs().doNotCacheIf( + "Only collects outputs of " + QUARKUS_BUILD_APP_TASK_NAME + " and " + QUARKUS_BUILD_DEP_TASK_NAME, + t -> !quarkusExt.getCacheLargeArtifacts().get()); }); TaskProvider imageBuild = tasks.register(IMAGE_BUILD_TASK_NAME, ImageBuild.class, buildConfig); @@ -242,7 +276,7 @@ private void registerTasks(Project project, QuarkusPluginExtension quarkusExt) { quarkusGenerateCode, quarkusGenerateCodeTests); }); - quarkusBuild.configure( + quarkusBuildApp.configure( task -> task.dependsOn(classesTask, resourcesTask, tasks.named(JavaPlugin.JAR_TASK_NAME))); SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); @@ -310,7 +344,7 @@ public void execute(Task task) { t.useJUnitPlatform(); }); // quarkusBuild is expected to run after the project has passed the tests - quarkusBuild.configure(task -> task.shouldRunAfter(tasks.withType(Test.class))); + quarkusBuildApp.configure(task -> task.shouldRunAfter(tasks.withType(Test.class))); SourceSet generatedSourceSet = sourceSets.create(QuarkusGenerateCode.QUARKUS_GENERATED_SOURCES); SourceSet generatedTestSourceSet = sourceSets.create(QuarkusGenerateCode.QUARKUS_TEST_GENERATED_SOURCES); diff --git a/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/extension/QuarkusPluginExtension.java b/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/extension/QuarkusPluginExtension.java index 499ca60d7a3cc2..01ef7f9a0734ce 100644 --- a/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/extension/QuarkusPluginExtension.java +++ b/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/extension/QuarkusPluginExtension.java @@ -1,12 +1,15 @@ package io.quarkus.gradle.extension; +import static io.quarkus.gradle.QuarkusPlugin.QUARKUS_PACKAGE_ADD_RUNNER_SUFFIX; +import static io.quarkus.gradle.QuarkusPlugin.QUARKUS_PACKAGE_OUTPUT_NAME; + import java.io.File; import java.nio.file.Path; import java.nio.file.Paths; import java.util.LinkedHashSet; +import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.Properties; import java.util.Set; import java.util.StringJoiner; import java.util.stream.Collectors; @@ -42,12 +45,20 @@ public class QuarkusPluginExtension { private final MapProperty quarkusBuildProperties; private final SourceSetExtension sourceSetExtension; + private final Property cacheLargeArtifacts; + private final Property cleanupBuildOutput; + public QuarkusPluginExtension(Project project) { this.project = project; finalName = project.getObjects().property(String.class); finalName.convention(project.provider(() -> String.format("%s-%s", project.getName(), project.getVersion()))); + this.cleanupBuildOutput = project.getObjects().property(Boolean.class) + .convention(true); + this.cacheLargeArtifacts = project.getObjects().property(Boolean.class) + .convention(!System.getenv().containsKey("CI")); + this.sourceSetExtension = new SourceSetExtension(); this.quarkusBuildProperties = project.getObjects().mapProperty(String.class, String.class); } @@ -96,34 +107,66 @@ public void beforeTest(Test task) { } } - public String buildNativeRunnerName(final Map taskSystemProps) { - Properties properties = new Properties(taskSystemProps.size()); - properties.putAll(taskSystemProps); - quarkusBuildProperties.get().entrySet() - .forEach(buildEntry -> properties.putIfAbsent(buildEntry.getKey(), buildEntry.getValue())); - System.getProperties().entrySet() - .forEach(propEntry -> properties.putIfAbsent(propEntry.getKey(), propEntry.getValue())); - System.getenv().entrySet().forEach( - envEntry -> properties.putIfAbsent(envEntry.getKey(), envEntry.getValue())); - StringBuilder nativeRunnerName = new StringBuilder(); - - if (properties.containsKey("quarkus.package.output-name")) { - nativeRunnerName.append(properties.get("quarkus.package.output-name")); - } else { - nativeRunnerName.append(finalName()); + public String resolveBuildProperty(String propertyKey, Map taskSystemProps, String defaultValue) { + Object v = taskSystemProps.get(propertyKey); + if (v instanceof String) { + return v.toString(); } - if (!properties.containsKey("quarkus.package.add-runner-suffix") - || (properties.containsKey("quarkus.package.add-runner-suffix") - && Boolean.parseBoolean((String) properties.get("quarkus.package.add-runner-suffix")))) { - nativeRunnerName.append("-runner"); + String s = quarkusBuildProperties.get().get(propertyKey); + if (s != null) { + return s; } - return nativeRunnerName.toString(); + s = System.getProperty(propertyKey); + if (s != null) { + return s; + } + s = System.getenv(propertyKey.toUpperCase(Locale.ROOT).replace('.', '_')); + if (s != null) { + return s; + } + return defaultValue; + } + + public String buildNativeRunnerBaseName(Map taskSystemProps) { + return resolveBuildProperty(QUARKUS_PACKAGE_OUTPUT_NAME, taskSystemProps, finalName()); + } + + public String buildNativeRunnerName(Map taskSystemProps) { + String outputName = buildNativeRunnerBaseName(taskSystemProps); + if (Boolean.parseBoolean(resolveBuildProperty(QUARKUS_PACKAGE_ADD_RUNNER_SUFFIX, taskSystemProps, "true"))) { + return outputName + "-runner"; + } + return outputName; } public Property getFinalName() { return finalName; } + /** + * Whether the build output, build/*-runner[.jar] and build/quarkus-app, for other package types than the + * currently configured one are removed, default is 'true'. + */ + public Property getCleanupBuildOutput() { + return cleanupBuildOutput; + } + + public void setCleanupBuildOutput(boolean cleanupBuildOutput) { + this.cleanupBuildOutput.set(cleanupBuildOutput); + } + + /** + * Whether large build artifacts, like uber-jar and native runners, are cached. Defaults to 'false' if the 'CI' environment + * variable is set, otherwise defaults to 'true'. + */ + public Property getCacheLargeArtifacts() { + return cacheLargeArtifacts; + } + + public void setCacheLargeArtifacts(boolean cacheLargeArtifacts) { + this.cacheLargeArtifacts.set(cacheLargeArtifacts); + } + public String finalName() { return getFinalName().get(); } diff --git a/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuild.java b/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuild.java index 1b3698e4f668a2..dd665c0a32e218 100644 --- a/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuild.java +++ b/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuild.java @@ -1,64 +1,35 @@ package io.quarkus.gradle.tasks; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Collections; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Properties; +import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.inject.Inject; import org.gradle.api.Action; import org.gradle.api.GradleException; -import org.gradle.api.file.FileCollection; -import org.gradle.api.java.archives.Attributes; -import org.gradle.api.provider.MapProperty; import org.gradle.api.tasks.CacheableTask; -import org.gradle.api.tasks.Classpath; -import org.gradle.api.tasks.Input; -import org.gradle.api.tasks.Internal; -import org.gradle.api.tasks.Optional; import org.gradle.api.tasks.OutputDirectory; import org.gradle.api.tasks.OutputFile; -import org.gradle.api.tasks.SourceSet; import org.gradle.api.tasks.TaskAction; import org.gradle.api.tasks.options.Option; -import io.quarkus.bootstrap.BootstrapException; -import io.quarkus.bootstrap.app.CuratedApplication; -import io.quarkus.bootstrap.app.QuarkusBootstrap; -import io.quarkus.bootstrap.model.ApplicationModel; -import io.quarkus.bootstrap.resolver.AppModelResolverException; import io.quarkus.gradle.dsl.Manifest; -import io.quarkus.maven.dependency.GACTV; import io.quarkus.runtime.util.StringUtil; @CacheableTask -public abstract class QuarkusBuild extends QuarkusTask { - - private static final String NATIVE_PROPERTY_NAMESPACE = "quarkus.native"; - private static final String MANIFEST_SECTIONS_PROPERTY_PREFIX = "quarkus.package.manifest.manifest-sections"; - private static final String MANIFEST_ATTRIBUTES_PROPERTY_PREFIX = "quarkus.package.manifest.attributes"; - - private static final String OUTPUT_DIRECTORY = "quarkus.package.output-directory"; - - private List ignoredEntries = new ArrayList<>(); - private Manifest manifest = new Manifest(); - - private Properties applicationProperties = new Properties(); +public abstract class QuarkusBuild extends QuarkusBuildTask { @Inject - public QuarkusBuild() { - super("Quarkus builds a runner jar based on the build jar"); - } - - public QuarkusBuild(String description) { - super(description); + @SuppressWarnings("CdiInjectionPointsInspection") + public QuarkusBuild(QuarkusBuildConfiguration buildConfiguration) { + super(buildConfiguration, "Quarkus builds a runner jar based on the build jar"); } public QuarkusBuild nativeArgs(Action> action) { @@ -70,159 +41,158 @@ public QuarkusBuild nativeArgs(Action> action) { return this; } - @Optional - @Input - public abstract MapProperty getForcedProperties(); - - @Optional - @Input - public List getIgnoredEntries() { - return ignoredEntries; + public QuarkusBuild manifest(Action action) { + action.execute(this.getManifest()); + return this; } @Option(description = "When using the uber-jar option, this option can be used to " + "specify one or more entries that should be excluded from the final jar", option = "ignored-entry") public void setIgnoredEntries(List ignoredEntries) { - this.ignoredEntries.addAll(ignoredEntries); - } - - @Classpath - public FileCollection getClasspath() { - SourceSet mainSourceSet = QuarkusGradleUtils.getSourceSet(getProject(), SourceSet.MAIN_SOURCE_SET_NAME); - return mainSourceSet.getCompileClasspath().plus(mainSourceSet.getRuntimeClasspath()) - .plus(mainSourceSet.getAnnotationProcessorPath()) - .plus(mainSourceSet.getResources()); - } - - @Input - public Map getQuarkusBuildSystemProperties() { - Map quarkusSystemProperties = new HashMap<>(); - for (Map.Entry systemProperty : System.getProperties().entrySet()) { - if (systemProperty.getKey().toString().startsWith("quarkus.") && - systemProperty.getValue() instanceof Serializable) { - quarkusSystemProperties.put(systemProperty.getKey(), systemProperty.getValue()); - } - } - return quarkusSystemProperties; - } - - @Input - public Map getQuarkusBuildEnvProperties() { - Map quarkusEnvProperties = new HashMap<>(); - for (Map.Entry systemProperty : System.getenv().entrySet()) { - if (systemProperty.getKey() != null && systemProperty.getKey().startsWith("QUARKUS_")) { - quarkusEnvProperties.put(systemProperty.getKey(), systemProperty.getValue()); - } - } - return quarkusEnvProperties; - } - - @Internal - public Manifest getManifest() { - return this.manifest; - } - - public QuarkusBuild manifest(Action action) { - action.execute(this.getManifest()); - return this; + buildConfiguration.ignoredEntries.addAll(ignoredEntries); } @OutputFile public File getRunnerJar() { - return new File(getProject().getBuildDir(), String.format("%s.jar", extension().buildNativeRunnerName(Map.of()))); + return effectiveConfig().runnerJar(); } @OutputFile public File getNativeRunner() { - return new File(getProject().getBuildDir(), extension().buildNativeRunnerName(Map.of())); + return effectiveConfig().nativeRunner(); } @OutputDirectory public File getFastJar() { - return new File(getProject().getBuildDir(), - this.getPropValueWithPrecedence(OUTPUT_DIRECTORY, java.util.Optional.of("quarkus-app"))); + return effectiveConfig().fastJar(); + } + + @OutputFile + public File getArtifactProperties() { + return new File(getProject().getBuildDir(), QUARKUS_ARTIFACT_PROPERTIES); } @TaskAction public void buildQuarkus() { - final ApplicationModel appModel; - final Map forcedProperties = getForcedProperties().getOrElse(Collections.emptyMap()); - - try { - appModel = extension().getAppModelResolver().resolveModel(new GACTV(getProject().getGroup().toString(), - getProject().getName(), getProject().getVersion().toString())); - } catch (AppModelResolverException e) { - throw new GradleException("Failed to resolve Quarkus application model for " + getProject().getPath(), e); + String packageType = effectiveConfig().packageType(); + + cleanup(); + + if (QuarkusBuildConfiguration.isFastJarPackageType(packageType)) { + assembleFastJar(); + } else if (QuarkusBuildConfiguration.isLegacyJarPackageType(packageType)) { + assembleLegacyJar(); + } else if (QuarkusBuildConfiguration.isMutableJarPackageType(packageType)) { + assembleFullBuild(); + } else if (QuarkusBuildConfiguration.isUberJarPackageType(packageType)) { + assembleFullBuild(); + } else { + throw new GradleException("Unsupported package type " + packageType); } + } - final Properties effectiveProperties = getBuildSystemProperties(appModel.getAppArtifact()); - effectiveProperties.putAll(forcedProperties); - if (ignoredEntries != null && ignoredEntries.size() > 0) { - String joinedEntries = String.join(",", ignoredEntries); - effectiveProperties.setProperty("quarkus.package.user-configured-ignored-entries", joinedEntries); + private void cleanup() { + if (extension().getCleanupBuildOutput().get()) { + // Caching and "up-to-date" checks depend on the inputs, this 'delete()' should ensure that the up-to-date + // checks work against "clean" outputs, considering that the outputs depend on the package-type. + getLogger().info("Removing potentially existing runner files and fast-jar directory."); + File fastJar = effectiveConfig().fastJar(); + getFileSystemOperations().delete(delete -> delete.delete(getRunnerJar(), getNativeRunner(), fastJar)); } + } - exportCustomManifestProperties(effectiveProperties); - - try (CuratedApplication appCreationContext = QuarkusBootstrap.builder() - .setBaseClassLoader(getClass().getClassLoader()) - .setExistingModel(appModel) - .setTargetDirectory(getProject().getBuildDir().toPath()) - .setBaseName(extension().finalName()) - .setBuildSystemProperties(effectiveProperties) - .setAppArtifact(appModel.getAppArtifact()) - .setLocalProjectDiscovery(false) - .setIsolateDeployment(true) - .build().bootstrap()) { - - // Processes launched from within the build task of Gradle (daemon) lose content - // generated on STDOUT/STDERR by the process (see https://github.com/gradle/gradle/issues/13522). - // We overcome this by letting build steps know that the STDOUT/STDERR should be explicitly - // streamed, if they need to make available that generated data. - // The io.quarkus.deployment.pkg.builditem.ProcessInheritIODisabled$Factory - // does the necessary work to generate such a build item which the build step(s) can rely on - appCreationContext.createAugmentor("io.quarkus.deployment.pkg.builditem.ProcessInheritIODisabled$Factory", - Collections.emptyMap()).createProductionApplication(); - - } catch (BootstrapException e) { - throw new GradleException("Failed to build a runnable JAR", e); + private void assembleLegacyJar() { + getLogger().info("Finalizing Quarkus build for {} packaging", effectiveConfig().packageType()); + + Path buildDir = getProject().getBuildDir().toPath(); + Path libDir = buildDir.resolve("lib"); + Path depBuildDir = effectiveConfig().depBuildDir(); + Path appBuildDir = effectiveConfig().appBuildDir(); + + getLogger().info("Removing potentially existing legacy-jar lib/ directory."); + getFileSystemOperations().delete(delete -> delete.delete(libDir)); + + getLogger().info("Copying lib/ directory from {} into {}", depBuildDir, buildDir); + getFileSystemOperations().copy(copy -> { + copy.into(buildDir); + copy.from(depBuildDir); + copy.include("lib/**"); + }); + + getLogger().info("Copying lib/ directory from {} into {}", appBuildDir, buildDir); + getFileSystemOperations().copy(copy -> { + copy.into(buildDir); + copy.from(appBuildDir); + copy.include("lib/**"); + }); + + // Quarkus' 'legacy-jar' package type produces 'lib/modified-*.jar' files for some dependencies. + // The following code block removes the non-modified jars. + getLogger().info("Cleaning up lib/ directory in {}", buildDir); + try (Stream files = Files.walk(libDir)) { + files.filter(Files::isRegularFile).filter(f -> f.getFileName().toString().startsWith("modified-")) + .map(f -> f.getParent().resolve(f.getFileName().toString().substring("modified-".length()))) + .collect(Collectors.toList()) // necessary :( + .forEach(f -> { + try { + Files.deleteIfExists(f); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } catch (IOException e) { + throw new GradleException("Failed to clean up non-modified jars in lib/"); } + + copyRunnersAndArtifactProperties(effectiveConfig().appBuildDir()); } - private void exportCustomManifestProperties(Properties buildSystemProperties) { - if (this.manifest == null) { - return; - } + private void assembleFullBuild() { + File targetDir = getProject().getBuildDir(); - for (Map.Entry attribute : manifest.getAttributes().entrySet()) { - buildSystemProperties.put(toManifestAttributeKey(attribute.getKey()), - attribute.getValue()); - } + // build/quarkus-build/gen + Path genBuildDir = effectiveConfig().genBuildDir(); - for (Map.Entry section : manifest.getSections().entrySet()) { - for (Map.Entry attribute : section.getValue().entrySet()) { - buildSystemProperties - .put(toManifestSectionAttributeKey(section.getKey(), attribute.getKey()), attribute.getValue()); - } - } + getLogger().info("Copying Quarkus build for {} packaging from {} into {}", effectiveConfig().packageType(), + genBuildDir, targetDir); + getFileSystemOperations().copy(copy -> { + copy.into(targetDir); + copy.from(genBuildDir); + }); + + copyRunnersAndArtifactProperties(effectiveConfig().genBuildDir()); } - private String toManifestAttributeKey(String key) { - if (key.contains("\"")) { - throw new GradleException("Manifest entry name " + key + " is invalid. \" characters are not allowed."); - } - return String.format("%s.\"%s\"", MANIFEST_ATTRIBUTES_PROPERTY_PREFIX, key); + private void assembleFastJar() { + File appTargetDir = effectiveConfig().fastJar(); + + // build/quarkus-build/app + Path appBuildBaseDir = effectiveConfig().appBuildDir(); + // build/quarkus-build/app/quarkus-app + Path appBuildDir = appBuildBaseDir.resolve(effectiveConfig().outputDirectory()); + // build/quarkus-build/dep + Path depBuildDir = effectiveConfig().depBuildDir(); + + getLogger().info("Synchronizing Quarkus build for {} packaging from {} and {} into {}", effectiveConfig().packageType(), + appBuildDir, depBuildDir, appTargetDir); + getFileSystemOperations().sync(sync -> { + sync.into(appTargetDir); + sync.from(appBuildDir, depBuildDir); + }); + + copyRunnersAndArtifactProperties(effectiveConfig().appBuildDir()); } - private String toManifestSectionAttributeKey(String section, String key) { - if (section.contains("\"")) { - throw new GradleException("Manifest section name " + section + " is invalid. \" characters are not allowed."); - } - if (key.contains("\"")) { - throw new GradleException("Manifest entry name " + key + " is invalid. \" characters are not allowed."); - } - return String.format("%s.\"%s\".\"%s\"", MANIFEST_SECTIONS_PROPERTY_PREFIX, section, - key); + private void copyRunnersAndArtifactProperties(Path sourceDir) { + File buildDir = getProject().getBuildDir(); + + getLogger().info("Copying remaining Quarkus application artifacts for {} packaging from {} into {}", + effectiveConfig().packageType(), sourceDir, buildDir); + getFileSystemOperations().copy( + copy -> copy.into(buildDir).from(sourceDir).include(QUARKUS_ARTIFACT_PROPERTIES, + effectiveConfig().nativeRunnerFileName(), + effectiveConfig().runnerJarFileName(), "jib-image*", + effectiveConfig().runnerBaseName() + "-native-image-source-jar/**")); } private String expandConfigurationKey(String shortKey) { @@ -232,40 +202,4 @@ private String expandConfigurationKey(String shortKey) { } return String.format("%s.%s", NATIVE_PROPERTY_NAMESPACE, hyphenatedKey); } - - private String getPropValueWithPrecedence(final String propName, final java.util.Optional defaultValue) { - if (applicationProperties.isEmpty()) { - SourceSet mainSourceSet = QuarkusGradleUtils.getSourceSet(getProject(), SourceSet.MAIN_SOURCE_SET_NAME); - - FileCollection configFiles = mainSourceSet.getResources() - .filter(file -> "application.properties".equalsIgnoreCase(file.getName())); - configFiles.forEach(file -> { - FileInputStream appPropsIS = null; - try { - appPropsIS = new FileInputStream(file.getAbsoluteFile()); - applicationProperties.load(appPropsIS); - appPropsIS.close(); - } catch (IOException e) { - if (appPropsIS != null) { - try { - appPropsIS.close(); - } catch (IOException ex) { - // Ignore exception closing. - } - } - } - }); - } - Map quarkusBuildProperties = extension().getQuarkusBuildProperties().get(); - if (quarkusBuildProperties.containsKey(propName)) { - return quarkusBuildProperties.get(propName); - } else if (applicationProperties.contains(propName)) { - return applicationProperties.getProperty(propName); - } else if (getQuarkusBuildEnvProperties().containsKey(propName)) { - return getQuarkusBuildEnvProperties().get(propName); - } else if (defaultValue.isPresent()) { - return defaultValue.get(); - } - return null; - } } diff --git a/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuildApp.java b/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuildApp.java new file mode 100644 index 00000000000000..5086a3ffb537ab --- /dev/null +++ b/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuildApp.java @@ -0,0 +1,202 @@ +package io.quarkus.gradle.tasks; + +import java.io.File; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; + +import javax.inject.Inject; + +import org.gradle.api.GradleException; +import org.gradle.api.java.archives.Attributes; +import org.gradle.api.logging.LogLevel; +import org.gradle.api.tasks.CacheableTask; +import org.gradle.api.tasks.OutputDirectory; +import org.gradle.api.tasks.TaskAction; + +import io.quarkus.bootstrap.BootstrapException; +import io.quarkus.bootstrap.app.CuratedApplication; +import io.quarkus.bootstrap.app.QuarkusBootstrap; +import io.quarkus.bootstrap.model.ApplicationModel; +import io.quarkus.bootstrap.resolver.AppModelResolverException; +import io.quarkus.gradle.QuarkusPlugin; +import io.quarkus.maven.dependency.GACTV; + +@CacheableTask +public abstract class QuarkusBuildApp extends QuarkusBuildTask { + + @Inject + @SuppressWarnings("CdiInjectionPointsInspection") + public QuarkusBuildApp(QuarkusBuildConfiguration buildConfiguration) { + super(buildConfiguration, "Quarkus builds a runner jar based on the build jar"); + } + + /** + * Points to {@code build/quarkus-build/app} and includes the uber-jar, native runner and "quarkus-app" directory + * w/o the `lib/` folder. + */ + @OutputDirectory + public File getAppBuildDir() { + return effectiveConfig().appBuildDir().toFile(); + } + + @TaskAction + public void finalizeQuarkusBuild() { + Path appDir = effectiveConfig().appBuildDir(); + + // Caching and "up-to-date" checks depend on the inputs, this 'delete()' should ensure that the up-to-date + // checks work against "clean" outputs, considering that the outputs depend on the package-type. + getFileSystemOperations().delete(delete -> delete.delete(appDir)); + + String packageType = effectiveConfig().packageType(); + + if (QuarkusBuildConfiguration.isFastJarPackageType(packageType)) { + fastJarBuild(); + } else if (QuarkusBuildConfiguration.isLegacyJarPackageType(packageType)) { + legacyJarBuild(); + } else if (QuarkusBuildConfiguration.isMutableJarPackageType(packageType)) { + generateBuild(); + } else if (QuarkusBuildConfiguration.isUberJarPackageType(packageType)) { + generateBuild(); + } else { + throw new GradleException("Unsupported package type " + packageType); + } + } + + private void legacyJarBuild() { + generateBuild(); + + Path genDir = effectiveConfig().genBuildDir(); + Path appDir = effectiveConfig().appBuildDir(); + + getLogger().info("Synchronizing Quarkus legacy-jar app for package type {} into {}", effectiveConfig().packageType(), + appDir); + + getFileSystemOperations().sync(sync -> { + sync.into(appDir); + sync.from(genDir); + sync.include("**", QuarkusPlugin.QUARKUS_ARTIFACT_PROPERTIES); + sync.exclude("lib/**"); + }); + getFileSystemOperations().copy(copy -> { + copy.into(appDir); + copy.from(genDir); + copy.include("lib/modified-*"); + }); + } + + private void fastJarBuild() { + generateBuild(); + + String outputDirectory = effectiveConfig().outputDirectory(); + Path genDir = effectiveConfig().genBuildDir(); + Path appDir = effectiveConfig().appBuildDir(); + + getLogger().info("Synchronizing Quarkus fast-jar-like app for package type {} into {}", effectiveConfig().packageType(), + appDir); + + getFileSystemOperations().sync(sync -> { + sync.into(appDir); + sync.from(genDir); + sync.exclude(outputDirectory + "/lib/**"); + }); + } + + void generateBuild() { + Path genDir = effectiveConfig().genBuildDir(); + String packageType = effectiveConfig().packageType(); + getLogger().info("Building Quarkus app for package type {} in {}", packageType, genDir); + + // Caching and "up-to-date" checks depend on the inputs, this 'delete()' should ensure that the up-to-date + // checks work against "clean" outputs, considering that the outputs depend on the package-type. + getFileSystemOperations().delete(delete -> delete.delete(genDir)); + + ApplicationModel appModel; + Map forcedProperties = getForcedProperties().getOrElse(Collections.emptyMap()); + + try { + appModel = extension().getAppModelResolver().resolveModel(new GACTV(getProject().getGroup().toString(), + getProject().getName(), getProject().getVersion().toString())); + } catch (AppModelResolverException e) { + throw new GradleException("Failed to resolve Quarkus application model for " + getProject().getPath(), e); + } + + final Properties effectiveProperties = getBuildSystemProperties(appModel.getAppArtifact()); + effectiveProperties.putAll(forcedProperties); + List ignoredEntries = getIgnoredEntries(); + if (!ignoredEntries.isEmpty()) { + String joinedEntries = String.join(",", ignoredEntries); + effectiveProperties.setProperty("quarkus.package.user-configured-ignored-entries", joinedEntries); + } + + exportCustomManifestProperties(effectiveProperties); + + getLogger().info("Starting Quarkus application build for package type {}", packageType); + + if (getLogger().isEnabled(LogLevel.DEBUG)) { + getLogger().debug("Effective properties: {}", + effectiveProperties.entrySet().stream().map(e -> "" + e) + .collect(Collectors.joining("\n ", "\n ", ""))); + } + + try (CuratedApplication appCreationContext = QuarkusBootstrap.builder() + .setBaseClassLoader(getClass().getClassLoader()) + .setExistingModel(appModel) + .setTargetDirectory(genDir) + .setBaseName(extension().finalName()) + .setBuildSystemProperties(effectiveProperties) + .setAppArtifact(appModel.getAppArtifact()) + .setLocalProjectDiscovery(false) + .setIsolateDeployment(true) + .build().bootstrap()) { + + // Processes launched from within the build task of Gradle (daemon) lose content + // generated on STDOUT/STDERR by the process (see https://github.com/gradle/gradle/issues/13522). + // We overcome this by letting build steps know that the STDOUT/STDERR should be explicitly + // streamed, if they need to make available that generated data. + // The io.quarkus.deployment.pkg.builditem.ProcessInheritIODisabled$Factory + // does the necessary work to generate such a build item which the build step(s) can rely on + appCreationContext.createAugmentor("io.quarkus.deployment.pkg.builditem.ProcessInheritIODisabled$Factory", + Collections.emptyMap()).createProductionApplication(); + + getLogger().debug("Quarkus application built successfully"); + } catch (BootstrapException e) { + throw new GradleException("Failed to build Quarkus application", e); + } + } + + private void exportCustomManifestProperties(Properties buildSystemProperties) { + for (Map.Entry attribute : effectiveConfig().manifest.getAttributes().entrySet()) { + buildSystemProperties.put(toManifestAttributeKey(attribute.getKey()), + attribute.getValue()); + } + + for (Map.Entry section : effectiveConfig().manifest.getSections().entrySet()) { + for (Map.Entry attribute : section.getValue().entrySet()) { + buildSystemProperties + .put(toManifestSectionAttributeKey(section.getKey(), attribute.getKey()), attribute.getValue()); + } + } + } + + private String toManifestAttributeKey(String key) { + if (key.contains("\"")) { + throw new GradleException("Manifest entry name " + key + " is invalid. \" characters are not allowed."); + } + return String.format("%s.\"%s\"", MANIFEST_ATTRIBUTES_PROPERTY_PREFIX, key); + } + + private String toManifestSectionAttributeKey(String section, String key) { + if (section.contains("\"")) { + throw new GradleException("Manifest section name " + section + " is invalid. \" characters are not allowed."); + } + if (key.contains("\"")) { + throw new GradleException("Manifest entry name " + key + " is invalid. \" characters are not allowed."); + } + return String.format("%s.\"%s\".\"%s\"", MANIFEST_SECTIONS_PROPERTY_PREFIX, section, + key); + } +} diff --git a/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuildConfiguration.java b/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuildConfiguration.java index 844feb76619a21..5c0e6ad17f5a3c 100644 --- a/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuildConfiguration.java +++ b/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuildConfiguration.java @@ -1,39 +1,389 @@ package io.quarkus.gradle.tasks; +import static io.quarkus.gradle.tasks.QuarkusGradleUtils.getSourceSet; + +import java.io.File; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; +import java.util.stream.Collectors; +import org.eclipse.microprofile.config.spi.ConfigSource; +import org.eclipse.microprofile.config.spi.ConfigSourceProvider; import org.gradle.api.Project; +import org.gradle.api.file.FileCollection; import org.gradle.api.provider.ListProperty; import org.gradle.api.provider.MapProperty; +import org.gradle.api.tasks.SourceSet; + +import io.quarkus.gradle.QuarkusPlugin; +import io.quarkus.gradle.dsl.Manifest; +import io.quarkus.gradle.extension.QuarkusPluginExtension; +import io.quarkus.runtime.configuration.ApplicationPropertiesConfigSourceLoader; +import io.smallrye.config.AbstractLocationConfigSourceLoader; +import io.smallrye.config.source.yaml.YamlConfigSource; public class QuarkusBuildConfiguration { + static final String QUARKUS_BUILD_DIR = "quarkus-build"; + static final String QUARKUS_BUILD_GEN_DIR = QUARKUS_BUILD_DIR + "/gen"; + static final String QUARKUS_BUILD_APP_DIR = QUARKUS_BUILD_DIR + "/app"; + static final String QUARKUS_BUILD_DEP_DIR = QUARKUS_BUILD_DIR + "/dep"; + + final Project project; + final QuarkusPluginExtension extension; + final SourceSet mainSourceSet; + final FileCollection classpath; + final ListProperty forcedDependenciesProperty; + final MapProperty forcedPropertiesProperty; + final List ignoredEntries = new ArrayList<>(); + final Manifest manifest = new Manifest(); - private final Project project; + // System properties starting with 'quarkus.' + private final Map quarkusSystemProperties; + // Environment properties starting with 'QUARKUS_' + private final Map quarkusEnvProperties; + // Gradle project + extension properties starting with 'quarkus.' - private ListProperty forcedDependencies; - private MapProperty forcedProperties; + private Map quarkusBuildProperties; + private Map forcedProperties; + private boolean effective; + private Map effectiveConfiguration; - public QuarkusBuildConfiguration(Project project) { + public QuarkusBuildConfiguration(Project project, QuarkusPluginExtension extension) { this.project = project; - forcedDependencies = project.getObjects().listProperty(String.class); - forcedProperties = project.getObjects().mapProperty(String.class, String.class); - } + this.extension = extension; - public ListProperty getForcedDependencies() { - return forcedDependencies; - } + mainSourceSet = getSourceSet(project, SourceSet.MAIN_SOURCE_SET_NAME); + classpath = dependencyClasspath(mainSourceSet); + + quarkusSystemProperties = collectQuarkusSystemProperties(); + quarkusEnvProperties = collectQuarkusEnvProperties(); - public void setForcedDependencies(List forcedDependencies) { - this.forcedDependencies.addAll(forcedDependencies); + forcedDependenciesProperty = project.getObjects().listProperty(String.class); + forcedPropertiesProperty = project.getObjects().mapProperty(String.class, String.class); } - public MapProperty getForcedProperties() { - return forcedProperties; + QuarkusBuildConfiguration effective() { + if (!effective) { + forcedProperties = forcedPropertiesProperty.get(); + quarkusBuildProperties = Collections + .unmodifiableMap(collectProperties(extension.getQuarkusBuildProperties().get())); + effectiveConfiguration = buildEffectiveConfiguration(); + effective = true; + } + return this; } public void setForcedProperties(Map forcedProperties) { this.forcedProperties.putAll(forcedProperties); } + + Path genBuildDir() { + return project.getBuildDir().toPath().resolve(QUARKUS_BUILD_GEN_DIR); + } + + Path appBuildDir() { + return project.getBuildDir().toPath().resolve(QUARKUS_BUILD_APP_DIR); + } + + Path depBuildDir() { + return project.getBuildDir().toPath().resolve(QUARKUS_BUILD_DEP_DIR); + } + + /** + * "final" location of the "fast-jar". + */ + File fastJar() { + return new File(project.getBuildDir(), outputDirectory()); + } + + /** + * "final" location of the "uber-jar". + */ + File runnerJar() { + return new File(project.getBuildDir(), runnerJarFileName()); + } + + /** + * "final" location of the "native" runner. + */ + File nativeRunner() { + return new File(project.getBuildDir(), nativeRunnerFileName()); + } + + String runnerJarFileName() { + return String.format("%s.jar", runnerName()); + } + + String nativeRunnerFileName() { + return runnerName(); + } + + String runnerName() { + return extension.buildNativeRunnerName(Map.of()); + } + + String runnerBaseName() { + return extension.buildNativeRunnerBaseName(Map.of()); + } + + String outputDirectory() { + return effectiveConfiguration().getOrDefault(QuarkusPlugin.OUTPUT_DIRECTORY, QuarkusPlugin.DEFAULT_OUTPUT_DIRECTORY); + } + + String packageType() { + return effectiveConfiguration().getOrDefault(QuarkusPlugin.QUARKUS_PACKAGE_TYPE, QuarkusPlugin.DEFAULT_PACKAGE_TYPE); + } + + Map getQuarkusSystemProperties() { + return quarkusSystemProperties; + } + + Map getQuarkusEnvProperties() { + return quarkusEnvProperties; + } + + static boolean isUberJarPackageType(String packageType) { + // Layout: + // build/ + return "uber-jar".equals(packageType); + } + + static boolean isLegacyJarPackageType(String packageType) { + // Layout: + // build/ + // build/lib/ + return "legacy-jar".equals(packageType); + } + + static boolean isMutableJarPackageType(String packageType) { + // like "fast-jar", but additional folder (not implemented, fallback to "full build" ATM). + // Additional folder: + // build//lib/deployment/ + // ^ contains dependency jars AND generated files + return "mutable-jar".equals(packageType); + } + + static boolean isFastJarPackageType(String packageType) { + // Layout: + // build// + // build//lib/boot/ + // build//lib/main/ + // build//quarkus/ + // build//app/ + // build//... + switch (packageType) { + case "jar": + case "fast-jar": + case "native": + return true; + default: + return false; + } + } + + private String quarkusProfile() { + String s = forcedProperties.get(QuarkusPlugin.QUARKUS_PROFILE); + if (s != null) { + project.getLogger().debug("Found profile name '{}' in forced properties", s); + return s; + } + s = quarkusSystemProperties.get(QuarkusPlugin.QUARKUS_PROFILE); + if (s != null) { + project.getLogger().debug("Found profile name '{}' in system properties", s); + return s; + } + s = quarkusEnvProperties.get(QuarkusPlugin.QUARKUS_PROFILE); + if (s != null) { + project.getLogger().debug("Found profile name '{}' in environment", s); + return s; + } + s = quarkusBuildProperties.get(QuarkusPlugin.QUARKUS_PROFILE); + if (s != null) { + project.getLogger().debug("Found profile name '{}' in build properties", s); + return s; + } + project.getLogger().debug("Using default profile '{}'", QuarkusPlugin.DEFAULT_PROFILE); + return QuarkusPlugin.DEFAULT_PROFILE; + } + + private Map buildEffectiveConfiguration() { + String profilePrefix = "%" + quarkusProfile() + "."; + + Map map = new HashMap<>(); + + Map projectProperties = collectProperties(project.getProperties()); + Map applicationProperties = loadApplicationProperties(); + + // TODO the previous implementation (asserted by e.g. BuildConfigurationTest) prefers configuration in + // application.properties over settings in the build file. Should that be changed to prefer build file + // settings over settings from application.properties? + addToEffectiveConfig(projectProperties, map); + addToEffectiveConfig(quarkusBuildProperties, map); + addToEffectiveConfig(applicationProperties, map); + addToEffectiveConfig(quarkusEnvProperties, map); + addToEffectiveConfig(quarkusSystemProperties, map); + addToEffectiveConfig(forcedProperties, map); + + addToEffectiveConfig(projectProperties, profilePrefix, map); + addToEffectiveConfig(quarkusBuildProperties, profilePrefix, map); + addToEffectiveConfig(applicationProperties, profilePrefix, map); + addToEffectiveConfig(quarkusEnvProperties, profilePrefix, map); + addToEffectiveConfig(quarkusSystemProperties, profilePrefix, map); + addToEffectiveConfig(forcedProperties, profilePrefix, map); + + if (project.getLogger().isInfoEnabled()) { + project.getLogger().info("Effective Quarkus application config: {}", + new TreeMap<>(map).entrySet().stream().map(Objects::toString) + .collect(Collectors.joining("\n ", "\n ", ""))); + } + + return Collections.unmodifiableMap(map); + } + + private static void addToEffectiveConfig(Map source, Map map) { + for (Map.Entry e : source.entrySet()) { + if (e.getValue() instanceof String) { + if (!e.getKey().startsWith("%")) { + map.put(e.getKey(), (String) e.getValue()); + } + } + } + } + + private static void addToEffectiveConfig(Map source, String profilePrefix, Map map) { + for (Map.Entry e : source.entrySet()) { + if (e.getValue() instanceof String) { + if (e.getKey().startsWith(profilePrefix)) { + map.put(e.getKey().substring(profilePrefix.length()), (String) e.getValue()); + } + } + } + } + + Map effectiveConfiguration() { + return effectiveConfiguration; + } + + private static Map collectProperties(Map source) { + return collectProperties(source, new HashMap<>()); + } + + private static Map collectProperties(Map source, Map target) { + source.forEach((k, v) -> { + String key = k.toString(); + if (key.startsWith("quarkus.") && v instanceof String) { + target.put(key, (String) v); + } + }); + return target; + } + + private static FileCollection dependencyClasspath(SourceSet mainSourceSet) { + return mainSourceSet.getCompileClasspath().plus(mainSourceSet.getRuntimeClasspath()) + .plus(mainSourceSet.getAnnotationProcessorPath()) + .plus(mainSourceSet.getResources()); + } + + private static Map collectQuarkusSystemProperties() { + return Collections.unmodifiableMap(collectProperties(System.getProperties())); + } + + private static Map collectQuarkusEnvProperties() { + Map quarkusEnvProperties = new HashMap<>(); + System.getenv().forEach((k, v) -> { + if (k.startsWith("QUARKUS_")) { + // convert environment name to property key + String key = k.toLowerCase(Locale.ROOT).replace('_', '.'); + quarkusEnvProperties.put(key, v); + } + }); + return Collections.unmodifiableMap(quarkusEnvProperties); + } + + private Map loadApplicationProperties() { + // TODO collection application.[properties|yaml] from dependencies ?? + + URL[] resourceUrls = mainSourceSet.getResources().getSourceDirectories().getFiles().stream().map(File::toURI) + .filter(uri -> "file".equals(uri.getScheme())) + .map(u -> { + try { + return u.toURL(); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + }) + .toArray(URL[]::new); + + Map config = new HashMap<>(); + + if (project.getLogger().isInfoEnabled()) { + project.getLogger().info("Loading Quarkus application config from resource URLs {}", + Arrays.stream(resourceUrls).map(Object::toString).collect(Collectors.joining(", "))); + } + + ClassLoader classLoader = new URLClassLoader(resourceUrls); + ConfigSourceProvider[] configSourceProviders = new ConfigSourceProvider[] { + new PropertiesConfigSourceProvider(), + new YamlConfigSourceProvider() + }; + for (ConfigSourceProvider configSourceProvider : configSourceProviders) { + project.getLogger().debug("Loading Quarkus application config via {}", configSourceProvider); + for (ConfigSource configSource : configSourceProvider.getConfigSources(classLoader)) { + Map properties = configSource.getProperties(); + project.getLogger().debug("Loaded {} Quarkus application config entries via {}", properties.size(), + configSource); + config.putAll(properties); + } + } + + if (project.getLogger().isDebugEnabled()) { + project.getLogger().debug("Loaded Quarkus application config from 'application.[properties|yaml|yml]: {}", + new TreeMap<>(config).entrySet().stream().map(Objects::toString) + .collect(Collectors.joining("\n ", "\n ", ""))); + } + + return Collections.unmodifiableMap(config); + } + + static final class YamlConfigSourceProvider extends AbstractLocationConfigSourceLoader implements ConfigSourceProvider { + @Override + protected String[] getFileExtensions() { + return new String[] { + "yaml", + "yml" + }; + } + + @Override + protected ConfigSource loadConfigSource(final URL url, final int ordinal) throws IOException { + return new YamlConfigSource(url, ordinal); + } + + @Override + public List getConfigSources(final ClassLoader classLoader) { + return loadConfigSources(new String[] { "application.yml", "application.yaml" }, 260, + classLoader); + } + } + + static final class PropertiesConfigSourceProvider extends ApplicationPropertiesConfigSourceLoader + implements ConfigSourceProvider { + @Override + public List getConfigSources(final ClassLoader classLoader) { + return loadConfigSources("application.properties", 260, classLoader); + } + } } diff --git a/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuildDependencies.java b/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuildDependencies.java new file mode 100644 index 00000000000000..8d156fbea91be6 --- /dev/null +++ b/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuildDependencies.java @@ -0,0 +1,162 @@ +package io.quarkus.gradle.tasks; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.inject.Inject; + +import org.gradle.api.GradleException; +import org.gradle.api.tasks.OutputDirectory; +import org.gradle.api.tasks.TaskAction; + +import io.quarkus.bootstrap.model.ApplicationModel; +import io.quarkus.gradle.QuarkusPlugin; +import io.quarkus.maven.dependency.ArtifactKey; +import io.quarkus.maven.dependency.DependencyFlags; +import io.quarkus.maven.dependency.ResolvedDependency; + +/** + * Collect the Quarkus app dependencies, the contents of the {@code quarkus-app/lib} folder, without making the task + * cache anything, but still provide up-to-date checks. + * + *

+ * Caching dependency jars is wasted effort and unnecessarily pollutes the Gradle build cache. + */ +public abstract class QuarkusBuildDependencies extends QuarkusBuildTask { + + @Inject + @SuppressWarnings("CdiInjectionPointsInspection") + public QuarkusBuildDependencies(QuarkusBuildConfiguration buildConfiguration) { + super(buildConfiguration, "Collect dependencies for the Quarkus application, prefer the 'quarkusBuild' task"); + } + + /** + * Points to {@code build/quarkus-build/dep}. + */ + @OutputDirectory + public File getDependenciesBuildDir() { + return effectiveConfig().depBuildDir().toFile(); + } + + @TaskAction + public void collectDependencies() { + Path depDir = effectiveConfig().depBuildDir(); + + // Caching and "up-to-date" checks depend on the inputs, this 'delete()' should ensure that the up-to-date + // checks work against "clean" outputs, considering that the outputs depend on the package-type. + getFileSystemOperations().delete(delete -> delete.delete(depDir)); + + String packageType = effectiveConfig().packageType(); + if (QuarkusBuildConfiguration.isFastJarPackageType(packageType)) { + fastJarDependencies(); + } else if (QuarkusBuildConfiguration.isLegacyJarPackageType(packageType)) { + legacyJarDependencies(); + } else if (QuarkusBuildConfiguration.isMutableJarPackageType(packageType)) { + getLogger().info( + "Falling back to 'full quarkus application build' for packaging type {}, this task's output is empty for this package type", + packageType); + } else if (QuarkusBuildConfiguration.isUberJarPackageType(packageType)) { + getLogger().info("Dependencies not needed for packaging type {}, this task's output is empty for this package type", + packageType); + } else { + throw new GradleException("Unsupported package type " + packageType); + } + } + + /** + * Resolves and copies dependencies for the {@code jar}, {@code fast-jar} and {@code native} package types. + * Does not work for {@code mutable-jar} ({@code lib/deployment/} missing). + * Unnecessary for {@code uber-jar}. + */ + private void fastJarDependencies() { + Path depDir = effectiveConfig().depBuildDir(); + Path libBoot = depDir.resolve("lib/boot"); + Path libMain = depDir.resolve("lib/main"); + jarDependencies(libBoot, libMain); + } + + /** + * Resolves and copies the dependencies for the {@code legacy-jar} package type. + * + *

+ * Side node: Quarkus' {@code legacy-jar} package type produces {@code modified-*.jar} files for some + * dependencies, but this implementation has no knowledge of which dependencies will be modified. + */ + private void legacyJarDependencies() { + Path depDir = effectiveConfig().depBuildDir(); + Path lib = depDir.resolve("lib"); + jarDependencies(lib, lib); + } + + private void jarDependencies(Path libBoot, Path libMain) { + Path depDir = effectiveConfig().depBuildDir(); + + getLogger().info("Placing Quarkus application dependencies for package type {} in {}", effectiveConfig().packageType(), + depDir); + + try { + Files.createDirectories(libBoot); + Files.createDirectories(libMain); + } catch (IOException e) { + throw new GradleException(String.format("Failed to create directories in %s", depDir), e); + } + + ApplicationModel appModel = extension().getApplicationModel(); + + // see https://quarkus.io/guides/class-loading-reference#configuring-class-loading + String removedArtifactsProp = effectiveConfig() + .effectiveConfiguration().getOrDefault(QuarkusPlugin.CLASS_LOADING_PARENT_FIRST_ARTIFACTS, ""); + java.util.Optional> optionalDependencies = java.util.Optional.ofNullable( + effectiveConfig().effectiveConfiguration().getOrDefault(QuarkusPlugin.CLASS_LOADING_REMOVED_ARTIFACTS, null)) + .map(s -> Arrays.stream(s.split(",")) + .map(String::trim) + .filter(gact -> !gact.isEmpty()) + .map(ArtifactKey::fromString) + .collect(Collectors.toSet())); + Set removedArtifacts = Arrays.stream(removedArtifactsProp.split(",")) + .map(String::trim) + .filter(gact -> !gact.isEmpty()) + .map(ArtifactKey::fromString) + .collect(Collectors.toSet()); + + appModel.getRuntimeDependencies().stream() + .filter(appDep -> { + // copied from io.quarkus.deployment.pkg.steps.JarResultBuildStep.includeAppDep + if (!appDep.isJar()) { + return false; + } + if (appDep.isOptional()) { + return optionalDependencies.map(appArtifactKeys -> appArtifactKeys.contains(appDep.getKey())) + .orElse(true); + } + return !removedArtifacts.contains(appDep.getKey()); + }) + .map(dep -> Map.entry(dep.isFlagSet(DependencyFlags.CLASSLOADER_RUNNER_PARENT_FIRST) ? libBoot : libMain, dep)) + .peek(depAndTarget -> { + ResolvedDependency dep = depAndTarget.getValue(); + Path targetDir = depAndTarget.getKey(); + dep.getResolvedPaths().forEach(p -> { + String file = dep.getGroupId() + '.' + p.getFileName(); + Path target = targetDir.resolve(file); + if (!Files.exists(target)) { + getLogger().debug("Dependency {} : copying {} to {}", + dep.toGACTVString(), + p, target); + try { + Files.copy(p, target); + } catch (IOException e) { + throw new GradleException(String.format("Failed to copy %s to %s", p, target), e); + } + } + }); + }) + .collect(Collectors.toMap(Map.Entry::getKey, depAndTarget -> 1, Integer::sum)) + .forEach((path, count) -> getLogger().info("Copied {} files into {}", count, path)); + } +} diff --git a/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuildTask.java b/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuildTask.java new file mode 100644 index 00000000000000..72a0085d40ad86 --- /dev/null +++ b/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusBuildTask.java @@ -0,0 +1,90 @@ +package io.quarkus.gradle.tasks; + +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import javax.inject.Inject; + +import org.gradle.api.file.FileCollection; +import org.gradle.api.file.FileSystemOperations; +import org.gradle.api.provider.MapProperty; +import org.gradle.api.tasks.Classpath; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.Optional; + +import io.quarkus.gradle.dsl.Manifest; +import io.quarkus.maven.dependency.ResolvedDependency; + +/** + * Base class for the {@link QuarkusBuildDependencies}, {@link QuarkusBuildApp}, {@link QuarkusBuild} tasks + */ +abstract class QuarkusBuildTask extends QuarkusTask { + + static final String NATIVE_PROPERTY_NAMESPACE = "quarkus.native"; + static final String MANIFEST_SECTIONS_PROPERTY_PREFIX = "quarkus.package.manifest.manifest-sections"; + static final String MANIFEST_ATTRIBUTES_PROPERTY_PREFIX = "quarkus.package.manifest.attributes"; + + static final String QUARKUS_ARTIFACT_PROPERTIES = "quarkus-artifact.properties"; + final QuarkusBuildConfiguration buildConfiguration; + + QuarkusBuildTask(QuarkusBuildConfiguration buildConfiguration, String description) { + super(description); + this.buildConfiguration = buildConfiguration; + } + + QuarkusBuildConfiguration effectiveConfig() { + return buildConfiguration.effective(); + } + + @Optional + @Input + public List getIgnoredEntries() { + return buildConfiguration.ignoredEntries; + } + + @Inject + protected abstract FileSystemOperations getFileSystemOperations(); + + @Optional + @Input + public abstract MapProperty getForcedProperties(); + + @Input + public Map getQuarkusBuildSystemProperties() { + return effectiveConfig().getQuarkusSystemProperties(); + } + + @Input + public Map getQuarkusBuildEnvProperties() { + return effectiveConfig().getQuarkusEnvProperties(); + } + + /** + * Retrieve all {@code quarkus.*} properties, which may be relevant for the Quarkus application build, from + * (likely) all possible sources. + */ + protected Properties getBuildSystemProperties(ResolvedDependency appArtifact) { + final Properties realProperties = new Properties(); + realProperties.putAll(effectiveConfig().effectiveConfiguration()); + realProperties.putIfAbsent("quarkus.application.name", appArtifact.getArtifactId()); + realProperties.putIfAbsent("quarkus.application.version", appArtifact.getVersion()); + return realProperties; + } + + @Classpath + public FileCollection getClasspath() { + return effectiveConfig().classpath; + } + + @Internal + public Manifest getManifest() { + return effectiveConfig().manifest; + } + + @Input + public Map getCachingRelevantInput() { + return effectiveConfig().effectiveConfiguration(); + } +} diff --git a/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusGenerateCode.java b/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusGenerateCode.java index c10b0b814a4e60..9d647ed21853a3 100644 --- a/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusGenerateCode.java +++ b/devtools/gradle/gradle-application-plugin/src/main/java/io/quarkus/gradle/tasks/QuarkusGenerateCode.java @@ -14,9 +14,12 @@ import org.gradle.api.GradleException; import org.gradle.api.artifacts.Configuration; +import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.CompileClasspath; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.OutputDirectory; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.SourceSet; import org.gradle.api.tasks.SourceSetContainer; import org.gradle.api.tasks.TaskAction; @@ -31,6 +34,7 @@ import io.quarkus.paths.PathList; import io.quarkus.runtime.LaunchMode; +@CacheableTask public class QuarkusGenerateCode extends QuarkusTask { public static final String QUARKUS_GENERATED_SOURCES = "quarkus-generated-sources"; @@ -66,6 +70,7 @@ public void setCompileClasspath(Configuration compileClasspath) { } @InputFiles + @PathSensitive(PathSensitivity.RELATIVE) public Set getInputDirectory() { Set inputDirectories = new HashSet<>(); diff --git a/devtools/gradle/gradle.properties b/devtools/gradle/gradle.properties index 2e90da2fcf509d..42de49e40cd24b 100644 --- a/devtools/gradle/gradle.properties +++ b/devtools/gradle/gradle.properties @@ -1,2 +1,3 @@ version = 2.16.999-SNAPSHOT kotlin_version = 1.7.22 +smallrye_config_version = 2.13.1 diff --git a/integration-tests/gradle/src/test/java/io/quarkus/gradle/nativeimage/NativeIntegrationTestIT.java b/integration-tests/gradle/src/test/java/io/quarkus/gradle/nativeimage/NativeIntegrationTestIT.java index b22b4eab2df071..58c4b50c26057f 100644 --- a/integration-tests/gradle/src/test/java/io/quarkus/gradle/nativeimage/NativeIntegrationTestIT.java +++ b/integration-tests/gradle/src/test/java/io/quarkus/gradle/nativeimage/NativeIntegrationTestIT.java @@ -16,7 +16,7 @@ public void nativeTestShouldRunIntegrationTest() throws Exception { BuildResult testResult = runGradleWrapper(projectDir, "clean", "testNative"); - assertThat(testResult.getTasks().get(":testNative")).isEqualTo(BuildResult.SUCCESS_OUTCOME); + assertThat(testResult.getTasks().get(":testNative")).isIn(BuildResult.SUCCESS_OUTCOME, BuildResult.FROM_CACHE); } @Test @@ -25,7 +25,7 @@ public void runNativeTestsWithOutputName() throws Exception { final BuildResult testResult = runGradleWrapper(projectDir, "clean", "testNative", "-Dquarkus.package.output-name=test"); - assertThat(testResult.getTasks().get(":testNative")).isEqualTo(BuildResult.SUCCESS_OUTCOME); + assertThat(testResult.getTasks().get(":testNative")).isIn(BuildResult.SUCCESS_OUTCOME, BuildResult.FROM_CACHE); } @Test @@ -34,7 +34,7 @@ public void runNativeTestsWithoutRunnerSuffix() throws Exception { final BuildResult testResult = runGradleWrapper(projectDir, "clean", "testNative", "-Dquarkus.package.add-runner-suffix=false"); - assertThat(testResult.getTasks().get(":testNative")).isEqualTo(BuildResult.SUCCESS_OUTCOME); + assertThat(testResult.getTasks().get(":testNative")).isIn(BuildResult.SUCCESS_OUTCOME, BuildResult.FROM_CACHE); } }