Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update Gradle to 5.5 version #4567

Merged
merged 21 commits into from
Aug 12, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,15 @@ buildscript {
}
dependencies {
classpath "cz.alenkacz:gradle-scalafmt:${gradle.scalafmt.version}"
classpath 'org.scoverage:gradle-scoverage:2.3.0'
}
}

plugins {
id "com.gradle.build-scan" version "1.14"
id "com.gradle.build-scan" version "2.3"
id "org.scoverage" version "3.1.5" apply false
}


buildScan {
termsOfServiceUrl = 'https://gradle.com/terms-of-service'
termsOfServiceAgree = 'yes'
Expand Down Expand Up @@ -102,6 +103,8 @@ def configureUnixClasspath(File script) {
// Get original line and append it
// with the configuration directory.
original += ':$APP_HOME/ext-lib/*:$APP_HOME/config'
//Ensure classes comes first. Used to refer to instrumented classes for code coverage
original = original.replace('CLASSPATH=', 'CLASSPATH=$APP_HOME/classes:')
}
}
.join('\n')
Expand Down
2 changes: 0 additions & 2 deletions common/scala/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,6 @@ dependencies {
exclude group: 'com.fasterxml.jackson.dataformat'
}
compile ('com.amazonaws:aws-java-sdk-cloudfront:1.11.517')
scoverage gradle.scoverage.deps

//Following constraints ensure that akka related transitive dependencies match the
//current version used
constraints {
Expand Down
9 changes: 4 additions & 5 deletions core/controller/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@ distDocker.dependsOn ':common:scala:distDocker', 'distTar'

project.archivesBaseName = "openwhisk-controller"

ext.coverageJars = [
"${buildDir}/libs/${project.archivesBaseName}-$version-scoverage.jar",
"${project(':common:scala').buildDir.absolutePath}/libs/openwhisk-common-$version-scoverage.jar"
ext.coverageDirs = [
"${buildDir}/classes/scala/scoverage",
"${project(':common:scala').buildDir.absolutePath}/classes/scala/scoverage"
]
distDockerCoverage.dependsOn ':common:scala:jarScoverage', 'jarScoverage'
distDockerCoverage.dependsOn ':common:scala:scoverageClasses', 'scoverageClasses'

repositories {
mavenCentral()
Expand All @@ -44,7 +44,6 @@ dependencies {
compile 'com.lightbend.akka.discovery:akka-discovery-marathon-api_2.12:0.11.0'
compile project(':common:scala')
compile project(':core:invoker')
scoverage gradle.scoverage.deps
}

tasks.withType(ScalaCompile) {
Expand Down
7 changes: 0 additions & 7 deletions core/cosmosdb/cache-invalidator/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,6 @@ distDocker.dependsOn ':common:scala:distDocker', 'distTar'

project.archivesBaseName = "openwhisk-cache-invalidator-cosmosdb"

ext.coverageJars = [
"${buildDir}/libs/${project.archivesBaseName}-$version-scoverage.jar",
"${project(':common:scala').buildDir.absolutePath}/libs/openwhisk-common-$version-scoverage.jar"
]
distDockerCoverage.dependsOn ':common:scala:jarScoverage', 'jarScoverage'

repositories {
mavenCentral()
}
Expand All @@ -45,7 +39,6 @@ dependencies {
exclude group: 'commons-logging'
}
compile 'com.typesafe.akka:akka-stream-kafka_2.12:1.0'
scoverage gradle.scoverage.deps
}

tasks.withType(ScalaCompile) {
Expand Down
9 changes: 4 additions & 5 deletions core/invoker/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@ apply from: '../../gradle/docker.gradle'
distDocker.dependsOn ':common:scala:distDocker', 'distTar'

project.archivesBaseName = "openwhisk-invoker"
ext.coverageJars = [
"${buildDir}/libs/${project.archivesBaseName}-$version-scoverage.jar",
"${project(':common:scala').buildDir.absolutePath}/libs/openwhisk-common-$version-scoverage.jar"
ext.coverageDirs = [
"${buildDir}/classes/scala/scoverage",
"${project(':common:scala').buildDir.absolutePath}/classes/scala/scoverage"
]
distDockerCoverage.dependsOn ':common:scala:jarScoverage', 'jarScoverage'
distDockerCoverage.dependsOn ':common:scala:scoverageClasses', 'scoverageClasses'

repositories {
mavenCentral()
Expand All @@ -48,7 +48,6 @@ dependencies {
exclude group: 'log4j'
exclude group: 'jline'
}
scoverage gradle.scoverage.deps
}

tasks.withType(ScalaCompile) {
Expand Down
1 change: 0 additions & 1 deletion core/standalone/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,5 @@ dependencies {
compile project(':core:controller')
compile project(':tools:admin')
compile 'org.rogach:scallop_2.12:3.3.1'
scoverage gradle.scoverage.deps
}

4 changes: 2 additions & 2 deletions gradle/docker.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,8 @@ task distDockerCoverage() {
//Copy the scoverage runtime jars
copy {from configurations.scoverage - configurations.compile; into "build/tmp/docker-coverage/ext-lib"}
//Copy the scoverage prepared jars
coverageJars.each {jar ->
copy {from file(jar); into "build/tmp/docker-coverage/lib"; rename { it.replace('-scoverage', '')}}
coverageDirs.each {dir ->
copy {from file(dir); into "build/tmp/docker-coverage/classes"}
}

def buildArgs = [
Expand Down
Binary file modified gradle/wrapper/gradle-wrapper.jar
Binary file not shown.
2 changes: 1 addition & 1 deletion gradle/wrapper/gradle-wrapper.properties
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,6 @@

distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.2-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-5.5.1-all.zip
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Switching to 5.5.1 and also to all mode so as to enable code assists in IDE

zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
2 changes: 1 addition & 1 deletion gradlew
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`

# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='-Dfile.encoding=UTF-8'
DEFAULT_JVM_OPTS='-Dfile.encoding=UTF-8 "-Xmx64m" "-Xms64m"'

# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"
Expand Down
2 changes: 1 addition & 1 deletion gradlew.bat
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%

@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS=-Dfile.encoding=UTF-8
set DEFAULT_JVM_OPTS=-Dfile.encoding=UTF-8 "-Xmx64m" "-Xms64m"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this too low?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If a different heap size is appropriate, we can change it. It's fine for us to diverge from the upstream gradlew[.bat] as needed.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For now build seems to work fine. If we see any issue then we can change. I believe Gradle wrapper internally forks a new jvm process so it needs bare minimum memory for its work


@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
Expand Down
7 changes: 0 additions & 7 deletions settings.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,6 @@ gradle.ext.scalafmt = [
config: new File(rootProject.projectDir, '.scalafmt.conf')
]

gradle.ext.scoverage = [
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now scoverage plugin internally adapts the classpath with required dependencies. So no need to explicitly add them

deps: [
'org.scoverage:scalac-scoverage-plugin_2.12:1.3.1',
'org.scoverage:scalac-scoverage-runtime_2.12:1.3.1'
]
]

gradle.ext.akka = [version : '2.5.22']
gradle.ext.akka_http = [version : '10.1.8']

Expand Down
85 changes: 37 additions & 48 deletions tests/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,14 @@ def leanExcludes = [
'invokerShoot/**'
]

def projectsWithCoverage = [
':common:scala',
':core:controller',
':core:invoker',
':tools:admin',
':core:cosmosdb:cache-invalidator'
]

def systemIncludes = [
"org/apache/openwhisk/core/limits/**",
"org/apache/openwhisk/core/admin/**",
Expand Down Expand Up @@ -186,6 +194,7 @@ dependencies {
compile "org.apache.curator:curator-test:${gradle.curator.version}"
compile 'com.atlassian.oai:swagger-request-validator-core:1.4.5'
compile 'com.typesafe.akka:akka-stream-kafka-testkit_2.12:1.0'
compile "com.typesafe.akka:akka-stream-testkit_2.12:${gradle.akka.version}"

compile "com.amazonaws:aws-java-sdk-s3:1.11.295"

Expand All @@ -195,10 +204,6 @@ dependencies {
compile project(':core:cosmosdb:cache-invalidator')
compile project(':tools:admin')



scoverage gradle.scoverage.deps

swaggerCodegen 'io.swagger:swagger-codegen-cli:2.3.1'
}

Expand Down Expand Up @@ -233,13 +238,13 @@ gradle.projectsEvaluated {
doFirst {
logTestSetInfo()
}
classpath = getScoverageClasspath(project)
classpath = getScoverageClasspath(project, projectsWithCoverage)
exclude getPattern(testSetName, "excludes")
include getPattern(testSetName, "includes")
}

task testCoverage(type: Test) {
classpath = getScoverageClasspath(project)
classpath = getScoverageClasspath(project, projectsWithCoverage)
}
tasks.withType(Test) {
systemProperties(System.getProperties())
Expand All @@ -250,8 +255,33 @@ gradle.projectsEvaluated {
showStandardStreams = true
exceptionFormat = 'full'
}
maxHeapSize = "1024m" //Gradle 5.5 defaults to 512MB which is low
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With Gradle 5.0 it now defaults to 512M of heap. Due to this OOM was seen in test runs on Travis (which has 7 GB memory). Hence specifying the size explicitly to 1024 MB now

All workers, including compilers and test executors, now start with 512MB of heap. The previous default was 1/4th of physical memory. Large projects may have to increase this setting on the relevant tasks, e.g. JavaCompile or Test.

outputs.upToDateWhen { false } // force tests to run every time
}
/**
* Task to generate coverage xml report. Requires the
* tests to be executed prior to its invocation
*/
task reportCoverage(type: ScoverageReport) {
Copy link
Member Author

@chetanmeh chetanmeh Aug 6, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The new version now creates task like report<Test Task Name>Scoverage as entry task. This task in turn adapts the compile task to use the instrumented classes. This works fine for conventional cases where tests are part of same module.

However in our case as we have a separate test module and further to capture coverage of code running within container (as part of integration test) we copy the coverage metrics files manually we cannot use default tasks.

Hence the report task duplicate bit of logic from actual task and clones the config from generated reportTestCoverage task

def dependentTasks = []
dependentTasks << copyMeasurementFiles
projectsWithCoverage.forEach {
dependentTasks << it + ':reportScoverage'
}
dependsOn(dependentTasks)
//Need to recreate the logic from
//https://github.com/scoverage/gradle-scoverage/blob/924bf49a8f981f119d0604b44a782f3f8eecb359/src/main/groovy/org/scoverage/ScoveragePlugin.groovy#L137
//default tasks retrigger the tests. As ours is a multi module integration
//test we have to adapt the classpath and hence cannot use default reportXXXScoverage tasks
runner = new org.scoverage.ScoverageRunner(project.configurations.scoverage)
reportDir = reportTestScoverage.reportDir
sources = reportTestScoverage.sources
dataDir = reportTestScoverage.dataDir
coverageOutputCobertura = reportTestScoverage.coverageOutputCobertura
coverageOutputXML = reportTestScoverage.coverageOutputXML
coverageOutputHTML = reportTestScoverage.coverageOutputHTML
coverageDebug = reportTestScoverage.coverageDebug
}
}

task copyMeasurementFiles() {
Expand All @@ -273,21 +303,6 @@ task copyMeasurementFiles() {
}
}

/**
* Task to generate coverage xml report. Requires the
* tests to be executed prior to its invocation
*/
task reportCoverage(type: ScoverageReport) {
dependsOn([
copyMeasurementFiles,
':common:scala:reportScoverage',
':core:controller:reportScoverage',
':core:invoker:reportScoverage',
':tools:admin:reportScoverage',
':core:cosmosdb:cache-invalidator:reportScoverage',
])

}

/**
* Scoverage measurement files are named like scoverage.measurements.xxx. Where xxx is thread id. While
Expand Down Expand Up @@ -315,37 +330,11 @@ def loadWhiskProps(){
p
}

/**
* Aggregates the scoverage xml reports from various modules into a
* single report
*/
task aggregateCoverage(type: JavaExec, dependsOn: reportCoverage) {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This task was never used as we rely on codecov to perform actual aggregation

//Taken from ScoverageAggregate
main = 'org.scoverage.AggregateReportApp'
classpath = project.extensions.scoverage.pluginClasspath
args = [
project.rootProject.projectDir, //Use the root project path so as to "see" all source paths
new File(project.buildDir, 'scoverage-aggregate'),
false, //Clean scoverage report post process
true, //coverageOutputCobertura
true, //coverageOutputXML
true, //coverageOutputHTML
false //coverageDebug
]
}

/**
* Prepares the classpath which refer to scoverage instrumented classes from
* dependent projects "before" the non instrumented classes
*/
def getScoverageClasspath(Project project) {
def projectNames = [
':common:scala',
':core:controller',
':core:invoker',
':tools:admin',
':core:cosmosdb:cache-invalidator'
]
def getScoverageClasspath(Project project, List<String> projectNames) {
def combinedClasspath = projectNames.inject(project.files([])) { result, name ->
def cp = project.project(name).sourceSets.scoverage.runtimeClasspath
result + cp.filter {it.name.contains('scoverage')}
Expand Down
1 change: 1 addition & 0 deletions tests/src/test/resources/logback-test.xml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

<!-- Kafka -->
<logger name="org.apache.kafka" level="ERROR" />
<logger name="kafka.server" level="ERROR" />

<!-- Zookeeper -->
<logger name="org.apache.zookeeper" level="ERROR" />
Expand Down
1 change: 0 additions & 1 deletion tools/admin/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -49,5 +49,4 @@ bootJar {
dependencies {
compile project(':common:scala')
compile 'org.rogach:scallop_2.12:3.1.2'
scoverage gradle.scoverage.deps
}