build: add windows aarch64 support (#2117)

* build: add windows aarch64 support

* Update windows.sh

* Update prepareWorkspace.sh

* Update windows.sh

* Update windows.sh

* Update jdk16_pipeline_config.groovy

* add support for cross compile

* use cross compiled version out job

* Update pipelines/build/common/cross_compiled_version_out.groovy

Co-authored-by: Morgan Davies <[email protected]>

* Update jdk16_pipeline_config.groovy

* fix version out for windows

* skip installer for aarch64 windows

* fix installers

Co-authored-by: Morgan Davies <[email protected]>

.github/workflows/build.yml

@@ -32,6 +32,7 @@ jobs:
       env:
         JAVA_TO_BUILD: ${{ matrix.version }}
         ARCHITECTURE: x64
+        CROSS_COMPILE: false
         VARIANT: ${{ matrix.vm }}
         TARGET_OS: ${{ matrix.os }}
         FILENAME: OpenJDK.tar.gz

build-farm/platform-specific-configurations/windows.sh

@@ -28,21 +28,27 @@ TOOLCHAIN_VERSION=""
 
 # Any version above 8 (11 for now due to openjdk-build#1409
 if [ "$JAVA_FEATURE_VERSION" -gt 11 ]; then
-    BOOT_JDK_VERSION="$((JAVA_FEATURE_VERSION-1))"
+    if [ "$ARCHITECTURE" == "aarch64" ]; then
+      # Windows aarch64 cross compiles requires same version boot jdk
+      BOOT_JDK_VERSION="$((JAVA_FEATURE_VERSION))"
+    else
+      BOOT_JDK_VERSION="$((JAVA_FEATURE_VERSION-1))"
+    fi
     BOOT_JDK_VARIABLE="JDK$(echo $BOOT_JDK_VERSION)_BOOT_DIR"
     if [ ! -d "$(eval echo "\$$BOOT_JDK_VARIABLE")" ]; then
       bootDir="$PWD/jdk-$BOOT_JDK_VERSION"
       # Note we export $BOOT_JDK_VARIABLE (i.e. JDKXX_BOOT_DIR) here
       # instead of BOOT_JDK_VARIABLE (no '$').
       export ${BOOT_JDK_VARIABLE}="$bootDir"
       if [ ! -d "$bootDir/bin" ]; then
-        echo "Downloading GA release of boot JDK version ${BOOT_JDK_VERSION}..."
-        releaseType="ga"
         # This is needed to convert x86-32 to x32 which is what the API uses
         case "$ARCHITECTURE" in
           "x86-32") downloadArch="x32";;
+          "aarch64") downloadArch="x64";;
           *) downloadArch="$ARCHITECTURE";;
         esac
+        echo "Downloading GA release of boot JDK version ${BOOT_JDK_VERSION}..."
+        releaseType="ga"
         apiUrlTemplate="https://api.adoptopenjdk.net/v3/binary/latest/\${BOOT_JDK_VERSION}/\${releaseType}/windows/\${downloadArch}/jdk/\${VARIANT}/normal/adoptopenjdk"
         apiURL=$(eval echo ${apiUrlTemplate})
         # make-adopt-build-farm.sh has 'set -e'. We need to disable that
@@ -194,6 +200,10 @@ then
   fi
 fi
 
+if [ "${ARCHITECTURE}" == "aarch64" ]; then
+  export CONFIGURE_ARGS_FOR_ANY_PLATFORM="${CONFIGURE_ARGS_FOR_ANY_PLATFORM} --disable-ccache --openjdk-target=aarch64-unknown-cygwin --with-build-jdk=$JDK_BOOT_DIR"
+fi
+
 if [ ! -z "${TOOLCHAIN_VERSION}" ]; then
     export CONFIGURE_ARGS_FOR_ANY_PLATFORM="${CONFIGURE_ARGS_FOR_ANY_PLATFORM} --with-toolchain-version=${TOOLCHAIN_VERSION}"
 fi

pipelines/build/common/build_base_file.groovy

@@ -99,6 +99,8 @@ class Builder implements Serializable {
 
         def additionalNodeLabels = formAdditionalBuildNodeLabels(platformConfig, variant)
 
+        def archLabel = getArchLabel(platformConfig, variant)
+
         def dockerImage = getDockerImage(platformConfig, variant)
 
         def dockerFile = getDockerFile(platformConfig, variant)
@@ -127,9 +129,10 @@ class Builder implements Serializable {
                 TEST_LIST: testList,
                 SCM_REF: scmReference,
                 BUILD_ARGS: buildArgs,
-                NODE_LABEL: "${additionalNodeLabels}&&${platformConfig.os}&&${platformConfig.arch}",
+                NODE_LABEL: "${additionalNodeLabels}&&${platformConfig.os}&&${archLabel}",
                 ACTIVE_NODE_TIMEOUT: activeNodeTimeout,
                 CODEBUILD: platformConfig.codebuild as Boolean,
+                CROSS_COMPILE: platformConfig.crossCompile as Boolean,
                 DOCKER_IMAGE: dockerImage,
                 DOCKER_FILE: dockerFile,
                 DOCKER_NODE: dockerNode,
@@ -241,6 +244,17 @@ class Builder implements Serializable {
         return overrideDocker
     }
 
+    def getArchLabel(Map<String, ?> configuration, String variant) {
+        def archLabelVal = ""
+        // Workaround for cross compiled architectures
+        if (configuration.containsKey("crossCompile")) {
+            archLabelVal = configuration.crossCompile
+        } else {
+            archLabelVal = configuration.arch
+        }
+        return archLabelVal
+    }
+
     /*
     Retrieves the dockerImage attribute from the build configurations.
     This specifies the DockerHub org and image to pull or build in case we don't have one stored in this repository.

pipelines/build/common/config_regeneration.groovy

@@ -97,6 +97,17 @@ class Regeneration implements Serializable {
         return configureArgs
     }
 
+    def getArchLabel(Map<String, ?> configuration, String variant) {
+        def archLabelVal = ""
+        // Workaround for cross compiled architectures
+        if (configuration.containsKey("crossCompile")) {
+            archLabelVal = configuration.crossCompile
+        } else {
+            archLabelVal = configuration.arch
+        }
+        return archLabelVal
+    }
+
     /*
     Retrieves the dockerImage attribute from the build configurations.
     This specifies the DockerHub org and image to pull or build in case we don't have one stored in this repository.
@@ -258,6 +269,8 @@ class Regeneration implements Serializable {
 
             def additionalNodeLabels = formAdditionalBuildNodeLabels(platformConfig, variant)
 
+            def archLabel = getArchLabel(platformConfig, variant)
+
             def dockerImage = getDockerImage(platformConfig, variant)
 
             def dockerFile = getDockerFile(platformConfig, variant)
@@ -276,9 +289,10 @@ class Regeneration implements Serializable {
                 TEST_LIST: testList,
                 SCM_REF: "",
                 BUILD_ARGS: buildArgs,
-                NODE_LABEL: "${additionalNodeLabels}&&${platformConfig.os}&&${platformConfig.arch}",
+                NODE_LABEL: "${additionalNodeLabels}&&${platformConfig.os}&&${archLabel}",
                 ACTIVE_NODE_TIMEOUT: "",
                 CODEBUILD: platformConfig.codebuild as Boolean,
+                CROSS_COMPILE: platformConfig.crossCompile as Boolean,
                 DOCKER_IMAGE: dockerImage,
                 DOCKER_FILE: dockerFile,
                 DOCKER_NODE: dockerNode,

pipelines/build/common/create_job_from_template.groovy

@@ -74,6 +74,7 @@ pipelineJob("$buildFolder/$JOB_NAME") {
                 <dt><strong>NODE_LABEL</strong></dt><dd>Labels of node to build on</dd>
                 <dt><strong>ACTIVE_NODE_TIMEOUT</strong></dt><dd>Number of minutes we will wait for a label-matching node to become active.</dd>
                 <dt><strong>CODEBUILD</strong></dt><dd>Use a dynamic codebuild machine if no other machine is available</dd>
+                <dt><strong>CROSS_COMPILE</strong></dt><dd>Set to true if this build is cross compiled</dd>
                 <dt><strong>DOCKER_IMAGE</strong></dt><dd>Use a docker build environment</dd>
                 <dt><strong>DOCKER_FILE</strong></dt><dd>Relative path to a dockerfile to be built and used on top of the DOCKER_IMAGE</dd>
                 <dt><strong>CONFIGURE_ARGS</strong></dt><dd>Arguments for ./configure. Escape all speech marks used within this parameter.</dd>

pipelines/build/common/riscv_version_out.groovy → pipelines/build/common/cross_compiled_version_out.groovy

@@ -1,21 +1,22 @@
 import org.jenkinsci.plugins.workflow.steps.FlowInterruptedException
 /**
- * This file is a jenkins job for extracting a version string from a cross-compiled, riscv binary.
+ * This file is a jenkins job for extracting a version string from a cross-compiled binary.
  * See https://github.com/AdoptOpenJDK/openjdk-build/issues/1773 for the inspiration for this.
  *
  * This file is referenced by the upstream job at pipelines/build/common/openjdk_build_pipeline.groovy.
- * This job is run at build-scripts/job/utils/job/riscv-version-out/.
+ * This job is run at build-scripts/job/utils/job/cross-compiled-version-out/.
  *
  * The job looks like:
- *  1. Switch into a riscv node
+ *  1. Switch into a suitable node
  *  2. Retrieve the artifacts built in the upstream job
  *  3. Run java -version and export to file
  *  4. Expose to the upstream job by archiving file
  */
 
 // TODO: ADD THE ACTIVE NODE TIMEOUT LOGIC HERE OR GET IT MERGED INTO JOB HELPER (https://github.com/AdoptOpenJDK/openjdk-build/issues/2235)
+String nodeLabel = (params.NODE) ?: ""
 
-node ("riscv&&ci.role.test") {
+node (nodeLabel) {
     timestamps {
         try {
             Integer JOB_TIMEOUT = 1
@@ -24,6 +25,8 @@ node ("riscv&&ci.role.test") {
                 String jobNumber = params.UPSTREAM_JOB_NUMBER ? params.UPSTREAM_JOB_NUMBER : ""
                 String jdkFileFilter = params.JDK_FILE_FILTER ? params.JDK_FILE_FILTER : ""
                 String fileName = params.FILENAME ? params.FILENAME : ""
+                String os = params.OS ? params.OS : ""
+
 
                 println "[INFO] PARAMS:"
                 println "UPSTREAM_JOB_NAME = ${jobName}"
@@ -38,8 +41,8 @@ node ("riscv&&ci.role.test") {
                     }
                 }
 
-                if (fileExists('RiscvVersionOuts')) {
-                    dir('RiscvVersionOuts') {
+                if (fileExists('CrossCompiledVersionOuts')) {
+                    dir('CrossCompiledVersionOuts') {
                         deleteDir()
                     }
                 }
@@ -58,7 +61,11 @@ node ("riscv&&ci.role.test") {
                     )
 
                     println "[INFO] Unzipping..."
-                    sh "tar -zxvf ${jdkFileFilter} && rm ${jdkFileFilter}"
+                    if (os == "windows") {
+                        sh "unzip ${jdkFileFilter} && rm ${jdkFileFilter}"
+                    } else {
+                        sh "tar -zxvf ${jdkFileFilter} && rm ${jdkFileFilter}"
+                    }
 
                     String jdkDir = sh(
                         script: "ls | grep jdk",
@@ -90,7 +97,7 @@ node ("riscv&&ci.role.test") {
                 }
 
                 // Write java version to file
-                dir ("RiscvVersionOuts") {
+                dir ("CrossCompiledVersionOuts") {
                     println "[INFO] Writing java version to ${fileName}..."
                     writeFile (
                         file: fileName,
@@ -100,8 +107,8 @@ node ("riscv&&ci.role.test") {
                 }
 
                 // Archive version.txt file
-                println "[INFO] Archiving RiscvVersionOuts/${fileName} to artifactory..."
-                archiveArtifacts artifacts: "RiscvVersionOuts/${fileName}"
+                println "[INFO] Archiving CrossCompiledVersionOuts/${fileName} to artifactory..."
+                archiveArtifacts artifacts: "CrossCompiledVersionOuts/${fileName}"
 
                 println "[SUCCESS] ${fileName} archived! Cleaning up..."
             }

pipelines/build/common/openjdk_build_pipeline.groovy

@@ -57,7 +57,7 @@ class Build {
     String j9Tags = ""
     String vendorName = ""
     String buildSource = ""
-    String riscvVersionPath = ""
+    String crossCompileVersionPath = ""
     Map variantVersion = [:]
 
     // Declare timeouts for each critical stage (unit is HOURS)
@@ -430,6 +430,12 @@ class Build {
         if (versionData.major == 8) {
             buildNumber = String.format("%02d", versionData.build)
         }
+
+        def INSTALLER_ARCH = "${buildConfig.ARCHITECTURE}"
+        // Wix toolset requires aarch64 builds to be called arm64
+        if (buildConfig.ARCHITECTURE == "aarch64") {
+            INSTALLER_ARCH = "arm64"
+        }
 
         // Get version patch number if one is present
         def patch_version = versionData.patch ?: 0
@@ -450,7 +456,7 @@ class Build {
                         context.string(name: 'PRODUCT_CATEGORY', value: "jdk"),
                         context.string(name: 'JVM', value: "${buildConfig.VARIANT}"),
                         context.string(name: 'SIGNING_CERTIFICATE', value: "${certificate}"),
-                        context.string(name: 'ARCH', value: "${buildConfig.ARCHITECTURE}"),
+                        context.string(name: 'ARCH', value: "${INSTALLER_ARCH}"),
                         ['$class': 'LabelParameterValue', name: 'NODE_LABEL', label: "${buildConfig.TARGET_OS}&&wix"]
                 ]
         context.copyArtifacts(
@@ -582,8 +588,8 @@ class Build {
 
             // Get Full Version Output
             String versionPath = "workspace/target/metadata/version.txt"
-            if (buildConfig.ARCHITECTURE.contains("riscv")) {
-                versionPath = riscvVersionPath
+            if (buildConfig.CROSS_COMPILE) {
+                versionPath = crossCompileVersionPath
             }
             context.println "INFO: Attempting to read ${versionPath}..."
 
@@ -830,11 +836,11 @@ class Build {
     }
 
     /*
-    Run the RISC-V version reader downstream job.
+    Run the cross comile version reader downstream job.
     In short, we archive the build artifacts to expose them to the job and run ./java version, copying the output back to here.
-    See riscv_version_out.groovy.
+    See cross_compiled_version_out.groovy.
     */
-    def readRiscvVersionString() {
+    def readCrossCompiledVersionString() {
         // Archive the artifacts early so we can copy them over to the downstream job
         try {
             context.timeout(time: buildTimeouts.BUILD_ARCHIVE_TIMEOUT, unit: "HOURS") {
@@ -849,26 +855,38 @@ class Build {
         String shortJobName = env.JOB_NAME.split('/').last()
         String copyFileFilter = "${shortJobName}_${env.BUILD_NUMBER}_version.txt"
 
-        def riscvJob = context.build job: "build-scripts/utils/riscv-version-out",
+        def nodeFilter = "${buildConfig.TARGET_OS}&&${buildConfig.ARCHITECTURE}"
+
+        def filter = ""
+
+        if (buildConfig.TARGET_OS == "windows") {
+            filter = "**\\OpenJDK*-jdk*_windows_*.zip"
+        } else {
+            filter = "**/OpenJDK*-jdk*.tar.gz"
+        }
+
+        def crossCompileVersionOut = context.build job: "build-scripts/utils/cross-compiled-version-out",
             propagate: true,
             parameters: [
                 context.string(name: 'UPSTREAM_JOB_NAME', value: "${env.JOB_NAME}"),
                 context.string(name: 'UPSTREAM_JOB_NUMBER', value: "${env.BUILD_NUMBER}"),
-                context.string(name: 'JDK_FILE_FILTER', value: "OpenJDK*-jdk*_linux_*.tar.gz"),
-                context.string(name: 'FILENAME', value: "${copyFileFilter}")
+                context.string(name: 'JDK_FILE_FILTER', value: "${filter}"),
+                context.string(name: 'FILENAME', value: "${copyFileFilter}"),
+                context.string(name: 'NODE', value: "${nodeFilter}"),
+                context.string(name: 'OS', value: "${buildConfig.TARGET_OS}")
             ]
 
         context.copyArtifacts(
-            projectName: "build-scripts/utils/riscv-version-out",
-            selector: context.specific("${riscvJob.getNumber()}"),
-            filter: "RiscvVersionOuts/${copyFileFilter}",
+            projectName: "build-scripts/utils/cross-compiled-version-out",
+            selector: context.specific("${crossCompileVersionOut.getNumber()}"),
+            filter: "CrossCompiledVersionOuts/${copyFileFilter}",
             target: "workspace/target/metadata",
             flatten: true
         )
 
         // We assign to a variable so it can be used in formMetadata() to find the correct version info
-        riscvVersionPath = "workspace/target/metadata/${copyFileFilter}"
-        return context.readFile(riscvVersionPath)
+        crossCompileVersionPath = "workspace/target/metadata/${copyFileFilter}"
+        return context.readFile(crossCompileVersionPath)
     }
 
     /*
@@ -938,9 +956,9 @@ class Build {
                     // Run a downstream job on riscv machine that returns the java version
                     // otherwise, just read the version.txt
                     String versionOut
-                    if (buildConfig.ARCHITECTURE.contains("riscv")) {
-                        context.println "[WARNING] Don't read faked version.txt on riscv! Archiving early and running downstream job on riscv machine to retrieve java version..."
-                        versionOut = readRiscvVersionString()
+                    if (buildConfig.CROSS_COMPILE) {
+                        context.println "[WARNING] Don't read faked version.txt on cross compiled build! Archiving early and running downstream job to retrieve java version..."
+                        versionOut = readCrossCompiledVersionString()
                     } else {
                         versionOut = context.readFile("workspace/target/metadata/version.txt")
                     }
@@ -952,8 +970,8 @@ class Build {
 
                 try {
                     context.timeout(time: buildTimeouts.BUILD_ARCHIVE_TIMEOUT, unit: "HOURS") {
-                        // We have already archived riscv artifacts, so only archive the metadata files
-                        if (buildConfig.ARCHITECTURE.contains("riscv")) {
+                        // We have already archived cross compiled artifacts, so only archive the metadata files
+                        if (buildConfig.CROSS_COMPILE) {
                             context.println "[INFO] Archiving JSON Files..."
                             context.archiveArtifacts artifacts: "workspace/target/*.json"
                         } else {

pipelines/jobs/configurations/jdk11u_pipeline_config.groovy

@@ -170,6 +170,7 @@ class Config11 {
                 os                   : 'linux',
                 dockerImage          : 'adoptopenjdk/centos6_build_image',
                 arch                 : 'riscv64',
+                crossCompile         : 'x64',
                 configureArgs        : '--disable-ddr --openjdk-target=riscv64-unknown-linux-gnu --with-sysroot=/opt/fedora28_riscv_root'
         ],
   ]

pipelines/jobs/configurations/jdk16.groovy

@@ -23,6 +23,9 @@ targetConfigurations = [
         "x64WindowsXL": [
                 "openj9"
         ],
+        "aarch64Windows" : [
+                "hotspot"
+        ],
         "x32Windows"  : [
                 "hotspot"
         ],

pipelines/jobs/configurations/jdk16_pipeline_config.groovy

@@ -86,6 +86,20 @@ class Config16 {
                 configureArgs        : '--with-noncompressedrefs'
         ],
 
+        aarch64Windows: [
+                os                  : 'windows',
+                arch                : 'aarch64',
+                crossCompile        : 'x64',
+                additionalNodeLabels: [
+                        hotspot: 'win2016&&vs2019'
+                ],
+                test                : [
+                        nightly: [],
+                        weekly : []
+                ]
+        ],
+
+
         x32Windows: [
                 os                  : 'windows',
                 arch                : 'x86-32',