[Backport] Improve build configuration time (#42674)

build.gradle

@@ -31,6 +31,7 @@ import org.gradle.plugins.ide.eclipse.model.SourceFolder
 plugins {
     id 'com.gradle.build-scan' version '2.2.1'
     id 'base'
+    id 'elasticsearch.global-build-info'
 }
 if (properties.get("org.elasticsearch.acceptScanTOS", "false") == "true") {
     buildScan {
@@ -263,7 +264,7 @@ allprojects {
   }
 
   project.afterEvaluate {
-    configurations.all {
+    configurations.matching { it.canBeResolved }.all {
       resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->
         projectSubstitutions.each { k,v ->
           subs.substitute(subs.module(k)).with(subs.project(v))
@@ -337,7 +338,7 @@ gradle.projectsEvaluated {
     if (tasks.findByPath('test') != null && tasks.findByPath('integTest') != null) {
       integTest.mustRunAfter test
     }
-    configurations.all { Configuration configuration ->
+    configurations.matching { it.canBeResolved }.all { Configuration configuration ->
       dependencies.all { Dependency dep ->
         Project upstreamProject = dependencyToProject(dep)
         if (upstreamProject != null) {
@@ -593,7 +594,3 @@ allprojects {
     }
   } 
 }
-
-
-
-

buildSrc/build.gradle

@@ -65,39 +65,10 @@ processResources {
 if (JavaVersion.current() < JavaVersion.VERSION_11) {
   throw new GradleException('At least Java 11 is required to build elasticsearch gradle tools')
 }
-// Gradle 4.10 does not support setting this to 11 yet
-targetCompatibility = "10"
-sourceCompatibility = "10"
-
-// We have a few classes that need to be compiled for older java versions because these are used to run checks against
-// those
-sourceSets {
-  minimumRuntime {
-    // We only want Java here, but the Groovy doesn't configure javadoc correctly if we don't define this as groovy
-    groovy {
-      srcDirs = ['src/main/minimumRuntime']
-    }
-  }
-}
-compileMinimumRuntimeGroovy {
-  // We can't use BuildPlugin here, so read from file
-  String minimumRuntimeVersion = file('src/main/resources/minimumRuntimeVersion').text.trim()
-  targetCompatibility = minimumRuntimeVersion
-  sourceCompatibility = minimumRuntimeVersion
-}
-dependencies {
-  if (project.ext.has("isEclipse") == false || project.ext.isEclipse == false) {
-      // eclipse is confused if this is set explicitly
-    compile sourceSets.minimumRuntime.output
-  }
-  minimumRuntimeCompile "junit:junit:${props.getProperty('junit')}"
-  minimumRuntimeCompile localGroovy()
-  minimumRuntimeCompile gradleApi()
-}
-jar {
-  from sourceSets.minimumRuntime.output
-}
 
+// Keep compatibility with Java 8 for external users of build-tools that haven't migrated to Java 11
+targetCompatibility = '8'
+sourceCompatibility = '8'
 
 /*****************************************************************************
  *                    Dependencies used by the entire build                  *
@@ -117,7 +88,7 @@ dependencies {
   compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
   compile 'org.apache.rat:apache-rat:0.11'
   compile "org.elasticsearch:jna:4.5.1"
-  compile 'com.github.jengelman.gradle.plugins:shadow:2.0.4'
+  compile 'com.github.jengelman.gradle.plugins:shadow:4.0.3'
   compile 'de.thetaphi:forbiddenapis:2.6'
   compile 'com.avast.gradle:gradle-docker-compose-plugin:0.8.12'
   testCompile "junit:junit:${props.getProperty('junit')}"
@@ -162,7 +133,6 @@ if (project != rootProject) {
   dependenciesInfo.enabled = false
   forbiddenApisMain.enabled = false
   forbiddenApisTest.enabled = false
-  forbiddenApisMinimumRuntime.enabled = false
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
 
@@ -184,16 +154,7 @@ if (project != rootProject) {
     from configurations.distribution
     into localDownloads
   }
-
-  test {
-    // The test task is configured to runtimeJava version, but build-tools doesn't support all of them, so test
-    // with compiler instead on the ones that are too old.
-    if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_10) {
-      executable = "${project.compilerJavaHome}/bin/java"
-    }
-  }
-
-  // This can't be an RandomizedTestingTask because we can't yet reference it
+
   task integTest(type: Test) {
     // integration test requires the local testing repo for example plugin builds
     dependsOn project.rootProject.allprojects.collect {

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -28,6 +28,7 @@ import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.test.RunTask
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin
 import org.gradle.api.InvalidUserDataException
+import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.publish.maven.MavenPublication
@@ -43,13 +44,13 @@ import java.util.regex.Pattern
 /**
  * Encapsulates build configuration for an Elasticsearch plugin.
  */
-class PluginBuildPlugin extends BuildPlugin {
+class PluginBuildPlugin implements Plugin<Project> {
 
     public static final String PLUGIN_EXTENSION_NAME = 'esplugin'
 
     @Override
     void apply(Project project) {
-        super.apply(project)
+        project.pluginManager.apply(BuildPlugin)
 
         PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project)
         configureDependencies(project)

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -116,32 +116,30 @@ class PrecommitTasks {
     }
 
     private static Task configureThirdPartyAudit(Project project) {
-        ThirdPartyAuditTask thirdPartyAuditTask = project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)
         ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
-        thirdPartyAuditTask.configure {
-            dependsOn(buildResources)
-            signatureFile = buildResources.copy("forbidden/third-party-audit.txt")
-            javaHome = project.runtimeJavaHome
-            targetCompatibility = project.runtimeJavaVersion
+        return project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class) { task ->
+            task.dependsOn(buildResources)
+            task.signatureFile = buildResources.copy("forbidden/third-party-audit.txt")
+            task.javaHome = project.runtimeJavaHome
+            task.targetCompatibility.set(project.provider({ project.runtimeJavaVersion }))
         }
-        return thirdPartyAuditTask
     }
 
     private static Task configureForbiddenApisCli(Project project) {
         project.pluginManager.apply(ForbiddenApisPlugin)
         ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
         project.tasks.withType(CheckForbiddenApis) {
             dependsOn(buildResources)
-            targetCompatibility = project.runtimeJavaVersion >= JavaVersion.VERSION_1_9 ?
-                    project.runtimeJavaVersion.getMajorVersion() : project.runtimeJavaVersion
-            if (project.runtimeJavaVersion > JavaVersion.VERSION_11) {
-                doLast {
+            doFirst {
+                // we need to defer this configuration since we don't know the runtime java version until execution time
+                targetCompatibility = project.runtimeJavaVersion.getMajorVersion()
+                if (project.runtimeJavaVersion > JavaVersion.VERSION_11) {
                     project.logger.info(
                             "Forbidden APIs does not support java version past 11. Will use the signatures from 11 for ",
                             project.runtimeJavaVersion
                     )
+                    targetCompatibility = JavaVersion.VERSION_11.getMajorVersion()
                 }
-                targetCompatibility = JavaVersion.VERSION_11.getMajorVersion()
             }
             bundledSignatures = [
                     "jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out"

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -317,12 +317,6 @@ class ClusterFormationTasks {
         // its run after plugins have been installed, as the extra config files may belong to plugins
         setup = configureExtraConfigFilesTask(taskName(prefix, node, 'extraConfig'), project, setup, node)
 
-        // If the node runs in a FIPS 140-2 JVM, the BCFKS default keystore will be password protected
-        if (project.inFipsJvm){
-            node.config.systemProperties.put('javax.net.ssl.trustStorePassword', 'password')
-            node.config.systemProperties.put('javax.net.ssl.keyStorePassword', 'password')
-        }
-
         // extra setup commands
         for (Map.Entry<String, Object[]> command : node.config.setupCommands.entrySet()) {
             // the first argument is the actual script name, relative to home
@@ -430,16 +424,17 @@ class ClusterFormationTasks {
         if (node.nodeVersion.major >= 7) {
             esConfig['indices.breaker.total.use_real_memory'] = false
         }
-        for (Map.Entry<String, Object> setting : node.config.settings) {
-            if (setting.value == null) {
-                esConfig.remove(setting.key)
-            } else {
-                esConfig.put(setting.key, setting.value)
-            }
-        }
 
         Task writeConfig = project.tasks.create(name: name, type: DefaultTask, dependsOn: setup)
         writeConfig.doFirst {
+            for (Map.Entry<String, Object> setting : node.config.settings) {
+                if (setting.value == null) {
+                    esConfig.remove(setting.key)
+                } else {
+                    esConfig.put(setting.key, setting.value)
+                }
+            }
+
             esConfig = configFilter.call(esConfig)
             File configFile = new File(node.pathConf, 'elasticsearch.yml')
             logger.info("Configuring ${configFile}")
@@ -760,6 +755,12 @@ class ClusterFormationTasks {
         }
         start.doLast(elasticsearchRunner)
         start.doFirst {
+            // If the node runs in a FIPS 140-2 JVM, the BCFKS default keystore will be password protected
+            if (project.inFipsJvm){
+                node.config.systemProperties.put('javax.net.ssl.trustStorePassword', 'password')
+                node.config.systemProperties.put('javax.net.ssl.keyStorePassword', 'password')
+            }
+
             // Configure ES JAVA OPTS - adds system properties, assertion flags, remote debug etc
             List<String> esJavaOpts = [node.env.get('ES_JAVA_OPTS', '')]
             String collectedSystemProperties = node.config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ")

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -86,50 +86,23 @@ class RestIntegTestTask extends DefaultTask {
         runner.include('**/*IT.class')
         runner.systemProperty('tests.rest.load_packaged', 'false')
 
-        /*
-         * We use lazy-evaluated strings in order to configure system properties whose value will not be known until
-         * execution time (e.g. cluster port numbers). Adding these via the normal DSL doesn't work as these get treated
-         * as task inputs and therefore Gradle attempts to snapshot them before/after task execution. This fails due
-         * to the GStrings containing references to non-serializable objects.
-         *
-         * We bypass this by instead passing this system properties vi a CommandLineArgumentProvider. This has the added
-         * side-effect that these properties are NOT treated as inputs, therefore they don't influence things like the
-         * build cache key or up to date checking.
-         */
-        def nonInputProperties = new CommandLineArgumentProvider() {
-            private final Map<String, Object> systemProperties = [:]
-
-            void systemProperty(String key, Object value) {
-                systemProperties.put(key, value)
-            }
-
-            @Override
-            Iterable<String> asArguments() {
-                return systemProperties.collect { key, value ->
-                    "-D${key}=${value.toString()}".toString()
-                }
-            }
-        }
-        runner.jvmArgumentProviders.add(nonInputProperties)
-        runner.ext.nonInputProperties = nonInputProperties
-
         if (System.getProperty("tests.rest.cluster") == null) {
             if (System.getProperty("tests.cluster") != null) {
                 throw new IllegalArgumentException("tests.rest.cluster and tests.cluster must both be null or non-null")
             }
             if (usesTestclusters == true) {
                 ElasticsearchCluster cluster = project.testClusters."${name}"
-                nonInputProperties.systemProperty('tests.rest.cluster', "${-> cluster.allHttpSocketURI.join(",") }")
-                nonInputProperties.systemProperty('tests.cluster', "${-> cluster.transportPortURI }")
+                runner.nonInputProperties.systemProperty('tests.rest.cluster', "${-> cluster.allHttpSocketURI.join(",") }")
+                runner.nonInputProperties.systemProperty('tests.cluster', "${-> cluster.transportPortURI }")
             } else {
                 // we pass all nodes to the rest cluster to allow the clients to round-robin between them
                 // this is more realistic than just talking to a single node
-                nonInputProperties.systemProperty('tests.rest.cluster', "${-> nodes.collect { it.httpUri() }.join(",")}")
-                nonInputProperties.systemProperty('tests.config.dir', "${-> nodes[0].pathConf}")
+                runner.nonInputProperties.systemProperty('tests.rest.cluster', "${-> nodes.collect { it.httpUri() }.join(",")}")
+                runner.nonInputProperties.systemProperty('tests.config.dir', "${-> nodes[0].pathConf}")
                 // TODO: our "client" qa tests currently use the rest-test plugin. instead they should have their own plugin
                 // that sets up the test cluster and passes this transport uri instead of http uri. Until then, we pass
                 // both as separate sysprops
-                nonInputProperties.systemProperty('tests.cluster', "${-> nodes[0].transportUri()}")
+                runner.nonInputProperties.systemProperty('tests.cluster', "${-> nodes[0].transportUri()}")
 
                 // dump errors and warnings from cluster log on failure
                 TaskExecutionAdapter logDumpListener = new TaskExecutionAdapter() {

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy

@@ -27,11 +27,14 @@ import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.gradle.api.InvalidUserDataException
+import org.gradle.api.JavaVersion
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.artifacts.Configuration
+import org.gradle.api.plugins.ExtraPropertiesExtension
 import org.gradle.api.plugins.JavaBasePlugin
 import org.gradle.api.plugins.JavaPlugin
+import org.gradle.api.plugins.JavaPluginExtension
 import org.gradle.api.tasks.SourceSet
 import org.gradle.api.tasks.SourceSetContainer
 import org.gradle.api.tasks.compile.JavaCompile
@@ -57,11 +60,14 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
         project.pluginManager.apply(JavaBasePlugin)
 
         project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask)
-        BuildPlugin.globalBuildInfo(project)
         BuildPlugin.configureRepositories(project)
         BuildPlugin.configureTestTasks(project)
         BuildPlugin.configureInputNormalization(project)
 
+        ExtraPropertiesExtension ext = project.extensions.getByType(ExtraPropertiesExtension)
+        project.extensions.getByType(JavaPluginExtension).sourceCompatibility = ext.get('minimumRuntimeVersion') as JavaVersion
+        project.extensions.getByType(JavaPluginExtension).targetCompatibility = ext.get('minimumRuntimeVersion') as JavaVersion
+
         // only setup tests to build
         SourceSetContainer sourceSets = project.extensions.getByType(SourceSetContainer)
         SourceSet testSourceSet = sourceSets.create('test')