Add OS/architecture classifier to distributions (#37881)

This commit adds classifiers to the distributions indicating the
OS (for archives) and platform. The current OSes are for windows, darwin (ie
macos) and linux. This change will allow future OS/architecture specific
changes to the distributions. Note the docs using distribution links
have been updated, but will be reworked in a followup to make OS
specific instructions for the archives.

buildSrc/build.gradle

@@ -183,8 +183,12 @@ if (project != rootProject) {
   }
 
   dependencies {
-    distribution project(':distribution:archives:zip')
-    distribution project(':distribution:archives:oss-zip')
+    distribution project(':distribution:archives:windows-zip')
+    distribution project(':distribution:archives:oss-windows-zip')
+    distribution project(':distribution:archives:darwin-tar')
+    distribution project(':distribution:archives:oss-darwin-tar')
+    distribution project(':distribution:archives:linux-tar')
+    distribution project(':distribution:archives:oss-linux-tar')
   }
 
   String localDownloads = "${rootProject.buildDir}/local-downloads"

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -191,25 +191,38 @@ class ClusterFormationTasks {
             throw new GradleException("Unknown distribution: ${distro} in project ${project.path}")
         }
         Version version = Version.fromString(elasticsearchVersion)
-        String group = "downloads.zip" // dummy group, does not matter except for integ-test-zip, it is ignored by the fake ivy repo
+        String os = getOs()
+        String classifier = "${os}-x86_64"
+        String packaging = os.equals('windows') ? 'zip' : 'tar.gz'
         String artifactName = 'elasticsearch'
         if (distro.equals('oss') && Version.fromString(elasticsearchVersion).onOrAfter('6.3.0')) {
             artifactName += '-oss'
         }
-        String snapshotProject = distro == 'oss' ? 'oss-zip' : 'zip'
         Object dependency
+        String snapshotProject = "${os}-${os.equals('windows') ? 'zip' : 'tar'}"
+        if (version.before("7.0.0")) {
+            snapshotProject = "zip"
+        }
+        if (distro.equals("oss")) {
+            snapshotProject = "oss-" + snapshotProject
+        }
         boolean internalBuild = project.hasProperty('bwcVersions')
         VersionCollection.UnreleasedVersionInfo unreleasedInfo = null
         if (project.hasProperty('bwcVersions')) {
             // NOTE: leniency is needed for external plugin authors using build-tools. maybe build the version compat info into build-tools?
             unreleasedInfo = project.bwcVersions.unreleasedInfo(version)
         }
         if (unreleasedInfo != null) {
-            dependency = project.dependencies.project(path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject)
+            dependency = project.dependencies.project(
+                        path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject)
         } else if (internalBuild && elasticsearchVersion.equals(VersionProperties.elasticsearch)) {
             dependency = project.dependencies.project(path: ":distribution:archives:${snapshotProject}")
         } else {
-            dependency = "${group}:${artifactName}:${elasticsearchVersion}@zip"
+            if (version.before('7.0.0')) {
+                classifier = "" // for bwc, before we had classifiers
+            }
+            // group does not matter as it is not used when we pull from the ivy repo that points to the download service
+            dependency = "dnm:${artifactName}:${elasticsearchVersion}${classifier}@${packaging}"
         }
         project.dependencies.add(configuration.name, dependency)
     }
@@ -335,8 +348,15 @@ class ClusterFormationTasks {
           the elasticsearch source tree then this should be the version of elasticsearch built by the source tree.
           If it isn't then Bad Things(TM) will happen. */
         Task extract = project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
-            from {
-                project.zipTree(configuration.singleFile)
+            if (getOs().equals("windows")) {
+                from {
+                    project.zipTree(configuration.singleFile)
+                }
+            } else {
+                // macos and linux use tar
+                from {
+                    project.tarTree(project.resources.gzip(configuration.singleFile))
+                }
             }
             into node.baseDir
         }
@@ -948,4 +968,15 @@ class ClusterFormationTasks {
         PluginPropertiesExtension extension = pluginProject.extensions.findByName('esplugin')
         return extension.name
     }
+
+    /** Find the current OS */
+    static String getOs() {
+        String os = "linux"
+        if (Os.FAMILY_WINDOWS) {
+            os = "windows"
+        } else if (Os.FAMILY_MAC) {
+            os = "darwin"
+        }
+        return os
+    }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -53,10 +53,10 @@ class VagrantTestPlugin implements Plugin<Project> {
 
     /** All distributions to bring into test VM, whether or not they are used **/
     static final List<String> DISTRIBUTIONS = unmodifiableList([
-            'archives:tar',
-            'archives:oss-tar',
-            'archives:zip',
-            'archives:oss-zip',
+            'archives:linux-tar',
+            'archives:oss-linux-tar',
+            'archives:windows-zip',
+            'archives:oss-windows-zip',
             'packages:rpm',
             'packages:oss-rpm',
             'packages:deb',

distribution/archives/build.gradle

@@ -105,13 +105,15 @@ task buildIntegTestZip(type: Zip) {
   with archiveFiles(transportModulesFiles, 'zip', true)
 }
 
-task buildZip(type: Zip) {
+task buildWindowsZip(type: Zip) {
   configure(commonZipConfig)
+  archiveClassifier = 'windows-x86_64'
   with archiveFiles(modulesFiles(false), 'zip', false)
 }
 
-task buildOssZip(type: Zip) {
+task buildOssWindowsZip(type: Zip) {
   configure(commonZipConfig)
+  archiveClassifier = 'windows-x86_64'
   with archiveFiles(modulesFiles(true), 'zip', true)
 }
 
@@ -122,13 +124,27 @@ Closure commonTarConfig = {
   fileMode 0644
 }
 
-task buildTar(type: Tar) {
+task buildDarwinTar(type: Tar) {
   configure(commonTarConfig)
+  archiveClassifier = 'darwin-x86_64'
   with archiveFiles(modulesFiles(false), 'tar', false)
 }
 
-task buildOssTar(type: Tar) {
+task buildOssDarwinTar(type: Tar) {
   configure(commonTarConfig)
+  archiveClassifier = 'darwin-x86_64'
+  with archiveFiles(modulesFiles(true), 'tar', true)
+}
+
+task buildLinuxTar(type: Tar) {
+  configure(commonTarConfig)
+  archiveClassifier = 'linux-x86_64'
+  with archiveFiles(modulesFiles(false), 'tar', false)
+}
+
+task buildOssLinuxTar(type: Tar) {
+  configure(commonTarConfig)
+  archiveClassifier = 'linux-x86_64'
   with archiveFiles(modulesFiles(true), 'tar', true)
 }
 

distribution/archives/oss-windows-zip/build.gradle

@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.

distribution/archives/windows-zip/build.gradle

@@ -0,0 +1,2 @@
+// This file is intentionally blank. All configuration of the
+// distribution is done in the parent project.

distribution/bwc/build.gradle

@@ -122,17 +122,34 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre
 
     Map<String, File> artifactFiles = [:]
     List<String> projectDirs = []
-    List<String> projects = ['zip', 'deb', 'rpm']
+    List<String> projects = ['deb', 'rpm']
+    if (bwcVersion.onOrAfter('7.0.0')) {
+      projects.addAll(['windows-zip', 'darwin-tar', 'linux-tar'])
+    } else {
+      projects.add('zip')
+    }
+
     for (String projectName : projects) {
         String baseDir = "distribution"
+        String classifier = ""
+        String extension = projectName
+        if (bwcVersion.onOrAfter('7.0.0') && (projectName.contains('zip') || projectName.contains('tar'))) {
+          int index = projectName.indexOf('-')
+          classifier = "-${projectName.substring(0, index)}-x86_64"
+          extension = projectName.substring(index + 1)
+          if (extension.equals('tar')) {
+            extension += '.gz'
+          }
+        }
         if (bwcVersion.onOrAfter('6.3.0')) {
-            baseDir += projectName == 'zip' ? '/archives' : '/packages'
+            baseDir += projectName.endsWith('zip') || projectName.endsWith('tar') ? '/archives' : '/packages'
             // add oss variant first
             projectDirs.add("${baseDir}/oss-${projectName}")
-            artifactFiles.put("oss-" + projectName, file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT.${projectName}"))
+            artifactFiles.put("oss-" + projectName, file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT${classifier}.${extension}"))
         }
         projectDirs.add("${baseDir}/${projectName}")
-        artifactFiles.put(projectName, file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT.${projectName}"))
+        artifactFiles.put(projectName,
+                file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT${classifier}.${extension}"))
     }
 
     Closure createRunBwcGradleTask = { name, extraConfig ->
@@ -216,9 +233,15 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre
         String artifactFileName = artifactFile.name
         String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
         String suffix = artifactFile.toString()[-3..-1]
+        int archIndex = artifactFileName.indexOf('x86_64')
+        String classifier = ''
+        if (archIndex != -1) {
+            int osIndex = artifactFileName.lastIndexOf('-', archIndex - 2)
+            classifier = "${artifactFileName.substring(osIndex + 1, archIndex - 1)}-x86_64"
+        }
         configurations.create(projectName)
         artifacts {
-            it.add(projectName, [file: artifactFile, name: artifactName, type: suffix, builtBy: buildBwcVersion])
+            it.add(projectName, [file: artifactFile, name: artifactName, classifier: classifier, type: suffix, builtBy: buildBwcVersion])
         }
     }
     // make sure no dependencies were added to assemble; we want it to be a no-op

distribution/docker/build.gradle

@@ -13,8 +13,8 @@ configurations {
 }
 
 dependencies {
-  dockerSource project(path: ":distribution:archives:tar")
-  ossDockerSource project(path: ":distribution:archives:oss-tar")
+  dockerSource project(path: ":distribution:archives:linux-tar")
+  ossDockerSource project(path: ":distribution:archives:oss-linux-tar")
 }
 
 ext.expansions = { oss ->

distribution/packages/build.gradle

@@ -101,8 +101,9 @@ Closure commonPackageConfig(String type, boolean oss) {
   return {
     dependsOn "process${oss ? 'Oss' : ''}${type.capitalize()}Files"
     packageName "elasticsearch${oss ? '-oss' : ''}"
+    arch (type == 'deb' ? 'amd64' : 'X86_64')
     // Follow elasticsearch's file naming convention
-    archiveName "${packageName}-${project.version}.${type}"
+    archiveName "${packageName}-${project.version}-${archString}.${type}"
 
     String prefix = "${oss ? 'oss-' : ''}${type}"
     destinationDir = file("${prefix}/build/distributions")
@@ -333,7 +334,6 @@ Closure commonRpmConfig(boolean oss) {
     packager 'Elasticsearch'
     version = project.version.replace('-', '_')
     release = '1'
-    arch 'NOARCH'
     os 'LINUX'
     distribution 'Elasticsearch'
     vendor 'Elasticsearch'

docs/reference/setup/install/deb.asciidoc

@@ -150,17 +150,17 @@ The Debian package for Elasticsearch v{version} can be downloaded from the websi
 
 ["source","sh",subs="attributes"]
 --------------------------------------------
-wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.deb
-wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.deb.sha512
-shasum -a 512 -c elasticsearch-{version}.deb.sha512 <1>
-sudo dpkg -i elasticsearch-{version}.deb
+wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-amd64.deb
+wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-amd64.deb.sha512
+shasum -a 512 -c elasticsearch-{version}-amd64.deb.sha512 <1>
+sudo dpkg -i elasticsearch-{version}-amd64.deb
 --------------------------------------------
 <1> Compares the SHA of the downloaded Debian package and the published checksum, which should output
-    `elasticsearch-{version}.deb: OK`.
+    `elasticsearch-{version}-amd64.deb: OK`.
 
 Alternatively, you can download the following package, which contains only 
 features that are available under the Apache 2.0 license: 
-https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}.deb    
+https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}-amd64.deb    
 
 endif::[]
 

docs/reference/setup/install/rpm.asciidoc

@@ -135,17 +135,17 @@ The RPM for Elasticsearch v{version} can be downloaded from the website and inst
 
 ["source","sh",subs="attributes"]
 --------------------------------------------
-wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.rpm
-wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.rpm.sha512
-shasum -a 512 -c elasticsearch-{version}.rpm.sha512 <1>
-sudo rpm --install elasticsearch-{version}.rpm
+wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-x86_64.rpm
+wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-x86_64.rpm.sha512
+shasum -a 512 -c elasticsearch-{version}-x86_64.rpm.sha512 <1>
+sudo rpm --install elasticsearch-{version}-x86_64.rpm
 --------------------------------------------
 <1> Compares the SHA of the downloaded RPM and the published checksum, which should output
-    `elasticsearch-{version}.rpm: OK`.
+    `elasticsearch-{version}-x86_64.rpm: OK`.
 
 Alternatively, you can download the following package, which contains only 
 features that are available under the Apache 2.0 license: 
-https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}.rpm    
+https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}-x86_64.rpm    
 
 endif::[]
 

docs/reference/setup/install/zip-targz.asciidoc

@@ -32,19 +32,19 @@ The `.zip` archive for Elasticsearch v{version} can be downloaded and installed
 
 ["source","sh",subs="attributes"]
 --------------------------------------------
-wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.zip
-wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.zip.sha512
-shasum -a 512 -c elasticsearch-{version}.zip.sha512 <1>
-unzip elasticsearch-{version}.zip
+wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-windows-x86_64.zip
+wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-windows-x86_64.zip.sha512
+shasum -a 512 -c elasticsearch-{version}-windows-x86_64.zip.sha512 <1>
+unzip elasticsearch-{version}-windows-x86_64.zip
 cd elasticsearch-{version}/ <2>
 --------------------------------------------
 <1> Compares the SHA of the downloaded `.zip` archive and the published checksum, which should output
-    `elasticsearch-{version}.zip: OK`.
+    `elasticsearch-{version}-windows-x86_64.zip: OK`.
 <2> This directory is known as `$ES_HOME`.
 
 Alternatively, you can download the following package, which contains only 
 features that are available under the Apache 2.0 license: 
-https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}.zip
+https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}-windows-x86_64.zip
 
 endif::[]
 
@@ -64,19 +64,19 @@ The `.tar.gz` archive for Elasticsearch v{version} can be downloaded and install
 
 ["source","sh",subs="attributes"]
 --------------------------------------------
-wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.tar.gz
-wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.tar.gz.sha512
-shasum -a 512 -c elasticsearch-{version}.tar.gz.sha512 <1>
-tar -xzf elasticsearch-{version}.tar.gz
+wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-linux-x86_64.tar.gz
+wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-linux-x86_64.tar.gz.sha512
+shasum -a 512 -c elasticsearch-{version}-linux-x86_64.tar.gz.sha512 <1>
+tar -xzf elasticsearch-{version}-linux-x86_64.tar.gz
 cd elasticsearch-{version}/ <2>
 --------------------------------------------
 <1> Compares the SHA of the downloaded `.tar.gz` archive and the published checksum, which should output
-    `elasticsearch-{version}.tar.gz: OK`.
+    `elasticsearch-{version}-linux-x86_64.tar.gz: OK`.
 <2> This directory is known as `$ES_HOME`.
 
 Alternatively, you can download the following package, which includes only 
 Apache 2.0 licensed code: 
-https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}.tar.gz
+https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}-linux-x86_64.tar.gz
 
 endif::[]
 

docs/reference/setup/install/zip-windows.asciidoc

@@ -31,11 +31,11 @@ endif::[]
 
 ifeval::["{release-state}"!="unreleased"]
 
-Download the `.zip` archive for Elasticsearch v{version} from: https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.zip
+Download the `.zip` archive for Elasticsearch v{version} from: https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-windows-x86_64.zip
 
 Alternatively, you can download the following package, which contains only 
 features that are available under the Apache 2.0 license: 
-https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}.zip
+https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}-windows-x86_64.zip
 
 Unzip it with your favourite unzip tool.  This will create a folder called
 +elasticsearch-{version}+, which we will refer to as `%ES_HOME%`. In a terminal

qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java

@@ -285,7 +285,7 @@ public void test90SecurityCliPackaging() {
         final Installation.Executables bin = installation.executables();
         final Shell sh = new Shell();
 
-        if (distribution().equals(Distribution.DEFAULT_TAR) || distribution().equals(Distribution.DEFAULT_ZIP)) {
+        if (distribution().equals(Distribution.DEFAULT_LINUX) || distribution().equals(Distribution.DEFAULT_WINDOWS)) {
             assertTrue(Files.exists(installation.lib.resolve("tools").resolve("security-cli")));
             Platforms.onLinux(() -> {
                 final Result result = sh.run(bin.elasticsearchCertutil + " help");
@@ -296,7 +296,7 @@ public void test90SecurityCliPackaging() {
                 final Result result = sh.run(bin.elasticsearchCertutil + " help");
                 assertThat(result.stdout, containsString("Simplifies certificate creation for use with the Elastic Stack"));
             });
-        } else if (distribution().equals(Distribution.OSS_TAR) || distribution().equals(Distribution.OSS_ZIP)) {
+        } else if (distribution().equals(Distribution.OSS_LINUX) || distribution().equals(Distribution.OSS_WINDOWS)) {
             assertFalse(Files.exists(installation.lib.resolve("tools").resolve("security-cli")));
         }
     }
@@ -312,7 +312,7 @@ public void test100ElasticsearchShardCliPackaging() {
             assertThat(result.stdout, containsString("A CLI tool to remove corrupted parts of unrecoverable shards"));
         };
 
-        if (distribution().equals(Distribution.DEFAULT_TAR) || distribution().equals(Distribution.DEFAULT_ZIP)) {
+        if (distribution().equals(Distribution.DEFAULT_LINUX) || distribution().equals(Distribution.DEFAULT_WINDOWS)) {
             Platforms.onLinux(action);
             Platforms.onWindows(action);
         }
@@ -330,7 +330,7 @@ public void test110ElasticsearchNodeCliPackaging() {
                     containsString("A CLI tool to unsafely recover a cluster after the permanent loss of too many master-eligible nodes"));
         };
 
-        if (distribution().equals(Distribution.DEFAULT_TAR) || distribution().equals(Distribution.DEFAULT_ZIP)) {
+        if (distribution().equals(Distribution.DEFAULT_LINUX) || distribution().equals(Distribution.DEFAULT_WINDOWS)) {
             Platforms.onLinux(action);
             Platforms.onWindows(action);
         }