Allow to use the bundled JDK in Elasticsearch

docs/command_line_reference.rst

@@ -362,6 +362,21 @@ Example::
 
    esrally --team-path=~/Projects/es-teams
 
+``target-os``
+~~~~~~~~~~~~~
+
+Specifies the name of the target operating system for which an artifact should be downloaded. By default this value is automatically derived based on the operating system Rally is run. This command line flag is only applicable to the ``download`` subcommand and allows to download an artifact for a different operating system. Example::
+
+    esrally download --distribution-version=7.5.1 --target-os=linux
+
+``target-arch``
+~~~~~~~~~~~~~~~
+
+Specifies the name of the target CPU architecture for which an artifact should be downloaded. By default this value is automatically derived based on the CPU architecture Rally is run. This command line flag is only applicable to the ``download`` subcommand and allows to download an artifact for a different CPU architecture. Example::
+
+    esrally download --distribution-version=7.5.1 --target-arch=x86_64
+
+
 ``car``
 ~~~~~~~
 
@@ -510,6 +525,8 @@ Example::
    # Force to run with JDK 7
    esrally --distribution-version=2.4.0 --runtime-jdk=7
 
+It is also possible to specify the JDK that is bundled with Elasticsearch with the special value ``bundled``. The `JDK is bundled from Elasticsearch 7.0.0 onwards <https://www.elastic.co/guide/en/elasticsearch/reference/current/release-highlights-7.0.0.html#_bundle_jdk_in_elasticsearch_distribution>`_.
+
 .. _clr_revision:
 
 ``revision``

esrally/mechanic/java_resolver.py

@@ -27,16 +27,23 @@ def determine_runtime_jdks():
         if override_runtime_jdk:
             return [override_runtime_jdk]
         else:
-            try:
-                return [int(v) for v in car_runtime_jdks.split(",")]
-            except ValueError:
-                raise exceptions.SystemSetupError(
-                    "Car config key \"runtime.jdk\" is invalid: \"{}\" (must be int)".format(car_runtime_jdks))
+            return allowed_runtime_jdks
 
     logger = logging.getLogger(__name__)
 
+    try:
+        allowed_runtime_jdks = [int(v) for v in car_runtime_jdks.split(",")]
+    except ValueError:
+        raise exceptions.SystemSetupError(
+            "Car config key \"runtime.jdk\" is invalid: \"{}\" (must be int)".format(car_runtime_jdks))
+
     runtime_jdk_versions = determine_runtime_jdks()
-    logger.info("Allowed JDK versions are %s.", runtime_jdk_versions)
-    major, java_home = jvm.resolve_path(runtime_jdk_versions)
-    logger.info("Detected JDK with major version [%s] in [%s].", major, java_home)
-    return major, java_home
+    if runtime_jdk_versions[0] == "bundled":
+        logger.info("Using JDK bundled with Elasticsearch.")
+        # assume that the bundled JDK is the highest available; the path is irrelevant
+        return allowed_runtime_jdks[0], None
+    else:
+        logger.info("Allowed JDK versions are %s.", runtime_jdk_versions)
+        major, java_home = jvm.resolve_path(runtime_jdk_versions)
+        logger.info("Detected JDK with major version [%s] in [%s].", major, java_home)
+        return major, java_home

esrally/mechanic/launcher.py

@@ -165,9 +165,10 @@ def _prepare_env(self, car_env, node_name, java_home, t):
         env = {}
         env.update(os.environ)
         env.update(car_env)
-        self._set_env(env, "PATH", os.path.join(java_home, "bin"), separator=os.pathsep, prepend=True)
-        # Don't merge here!
-        env["JAVA_HOME"] = java_home
+        if java_home:
+            self._set_env(env, "PATH", os.path.join(java_home, "bin"), separator=os.pathsep, prepend=True)
+            # Don't merge here!
+            env["JAVA_HOME"] = java_home
         env["ES_JAVA_OPTS"] = "-XX:+ExitOnOutOfMemoryError"
 
         # we just blindly trust telemetry here...

esrally/mechanic/provisioner.py

@@ -25,15 +25,20 @@
 
 from esrally import exceptions
 from esrally.mechanic import team, java_resolver
-from esrally.utils import console, io, process, versions
+from esrally.utils import console, convert, io, process, versions
 
 
 def local(cfg, car, plugins, cluster_settings, ip, http_port, all_node_ips, all_node_names, target_root, node_name):
     distribution_version = cfg.opts("mechanic", "distribution.version", mandatory=False)
 
     node_root_dir = os.path.join(target_root, node_name)
 
-    _, java_home = java_resolver.java_home(car.mandatory_var("runtime.jdk"), cfg)
+    runtime_jdk_bundled = convert.to_bool(car.mandatory_var("runtime.jdk.bundled"))
+    if runtime_jdk_bundled:
+        java_home = None
+    else:
+        runtime_jdk = car.mandatory_var("runtime.jdk")
+        _, java_home = java_resolver.java_home(runtime_jdk, cfg)
 
     es_installer = ElasticsearchInstaller(car, java_home, node_name, node_root_dir, all_node_ips, all_node_names, ip, http_port)
     plugin_installers = [PluginInstaller(plugin, java_home) for plugin in plugins]
@@ -268,7 +273,10 @@ def delete_pre_bundled_configuration(self):
         shutil.rmtree(config_path)
 
     def invoke_install_hook(self, phase, variables):
-        self.hook_handler.invoke(phase.name, variables=variables, env={"JAVA_HOME": self.java_home})
+        env = {}
+        if self.java_home:
+            env["JAVA_HOME"] = self.java_home
+        self.hook_handler.invoke(phase.name, variables=variables, env=env)
 
     @property
     def variables(self):
@@ -334,7 +342,7 @@ def install(self, es_home_path, plugin_url=None):
             self.logger.info("Installing [%s] into [%s]", self.plugin_name, es_home_path)
             install_cmd = '%s install --batch "%s"' % (installer_binary_path, self.plugin_name)
 
-        return_code = process.run_subprocess_with_logging(install_cmd, env={"JAVA_HOME": self.java_home})
+        return_code = process.run_subprocess_with_logging(install_cmd, env=self.env())
         # see: https://www.elastic.co/guide/en/elasticsearch/plugins/current/_other_command_line_parameters.html
         if return_code == 0:
             self.logger.info("Successfully installed [%s].", self.plugin_name)
@@ -348,7 +356,13 @@ def install(self, es_home_path, plugin_url=None):
                                         (self.plugin_name, str(return_code)))
 
     def invoke_install_hook(self, phase, variables):
-        self.hook_handler.invoke(phase.name, variables=variables, env={"JAVA_HOME": self.java_home})
+        self.hook_handler.invoke(phase.name, variables=variables, env=self.env())
+
+    def env(self):
+        env = {}
+        if self.java_home:
+            env["JAVA_HOME"] = self.java_home
+        return env
 
     @property
     def variables(self):

esrally/mechanic/supplier.py

@@ -37,21 +37,31 @@ def create(cfg, sources, distribution, build, car, plugins=None):
     distribution_version = cfg.opts("mechanic", "distribution.version", mandatory=False)
     supply_requirements = _supply_requirements(sources, distribution, build, plugins, revisions, distribution_version)
     build_needed = any([build for _, _, build in supply_requirements.values()])
+    es_supplier_type, es_version, es_build = supply_requirements["elasticsearch"]
     src_config = cfg.all_opts("source")
     suppliers = []
 
+    target_os = cfg.opts("mechanic", "target.os", mandatory=False)
+    target_arch = cfg.opts("mechanic", "target.arch", mandatory=False)
+    template_renderer = TemplateRenderer(version=es_version, os_name=target_os, arch=target_arch)
+
     if build_needed:
         java_home = _java_home(car)
         es_src_dir = os.path.join(_src_dir(cfg), _config_value(src_config, "elasticsearch.src.subdir"))
         builder = Builder(es_src_dir, java_home, paths.logs())
     else:
         builder = None
 
-    es_supplier_type, es_version, es_build = supply_requirements["elasticsearch"]
     if es_supplier_type == "source":
         es_src_dir = os.path.join(_src_dir(cfg), _config_value(src_config, "elasticsearch.src.subdir"))
         suppliers.append(
-            ElasticsearchSourceSupplier(es_version, es_src_dir, remote_url=cfg.opts("source", "remote.repo.url"), car=car, builder=builder))
+            ElasticsearchSourceSupplier(es_version,
+                                        es_src_dir,
+                                        remote_url=cfg.opts("source", "remote.repo.url"),
+                                        car=car,
+                                        builder=builder,
+                                        template_renderer=template_renderer)
+        )
         repo = None
     else:
         es_src_dir = None
@@ -67,8 +77,8 @@ def create(cfg, sources, distribution, build, car, plugins=None):
         dist_cfg.update(cfg.all_opts("distributions"))
         repo = DistributionRepository(name=cfg.opts("mechanic", "distribution.repository"),
                                       distribution_config=dist_cfg,
-                                      version=es_version)
-        suppliers.append(ElasticsearchDistributionSupplier(repo, distributions_root))
+                                      template_renderer=template_renderer)
+        suppliers.append(ElasticsearchDistributionSupplier(repo, es_version, distributions_root))
 
     for plugin in plugins:
         supplier_type, plugin_version, build_plugin = supply_requirements[plugin.name]
@@ -174,6 +184,30 @@ def _src_dir(cfg, mandatory=True):
                                           " all prerequisites and reconfigure Rally with %s configure" % PROGRAM_NAME)
 
 
+class TemplateRenderer:
+    def __init__(self, version, os_name=None, arch=None):
+        self.version = version
+        if os_name is not None:
+            self.os = os_name
+        else:
+            self.os = sysstats.os_name().lower()
+        if arch is not None:
+            self.arch = arch
+        else:
+            self.arch = sysstats.cpu_arch().lower()
+
+    def render(self, template):
+        substitutions = {
+            "{{VERSION}}": self.version,
+            "{{OSNAME}}": self.os,
+            "{{ARCH}}": self.arch
+        }
+        r = template
+        for key, replacement in substitutions.items():
+            r = r.replace(key, replacement)
+        return r
+
+
 class CompositeSupplier:
     def __init__(self, suppliers):
         self.suppliers = suppliers
@@ -190,26 +224,32 @@ def __call__(self, *args, **kwargs):
 
 
 class ElasticsearchSourceSupplier:
-    def __init__(self, revision, es_src_dir, remote_url, car, builder):
+    def __init__(self, revision, es_src_dir, remote_url, car, builder, template_renderer):
         self.revision = revision
         self.src_dir = es_src_dir
         self.remote_url = remote_url
         self.car = car
         self.builder = builder
+        self.template_renderer = template_renderer
 
     def fetch(self):
         SourceRepository("Elasticsearch", self.remote_url, self.src_dir).fetch(self.revision)
 
     def prepare(self):
         if self.builder:
-            self.builder.build([self.car.mandatory_var("clean_command"), self.car.mandatory_var("build_command")])
+            self.builder.build([
+                self.template_renderer.render(self.car.mandatory_var("clean_command")),
+                self.template_renderer.render(self.car.mandatory_var("system.build_command"))
+            ])
 
     def add(self, binaries):
         binaries["elasticsearch"] = self.resolve_binary()
 
     def resolve_binary(self):
         try:
-            return glob.glob("{}/{}".format(self.src_dir, self.car.mandatory_var("artifact_path_pattern")))[0]
+            path = os.path.join(self.src_dir,
+                                self.template_renderer.render(self.car.mandatory_var("system.artifact_path_pattern")))
+            return glob.glob(path)[0]
         except IndexError:
             raise SystemSetupError("Couldn't find a tar.gz distribution. Please run Rally with the pipeline 'from-sources-complete'.")
 
@@ -295,9 +335,9 @@ def resolve_binary(self):
 
 
 class ElasticsearchDistributionSupplier:
-    def __init__(self, repo, distributions_root):
+    def __init__(self, repo, version, distributions_root):
         self.repo = repo
-        self.version = repo.version
+        self.version = version
         self.distributions_root = distributions_root
         # will be defined in the prepare phase
         self.distribution_path = None
@@ -484,15 +524,19 @@ def run(self, command, override_src_dir=None):
 
 
 class DistributionRepository:
-    def __init__(self, name, distribution_config, version):
+    def __init__(self, name, distribution_config, template_renderer):
         self.name = name
         self.cfg = distribution_config
-        self.version = version
+        self.runtime_jdk_bundled = convert.to_bool(self.cfg.get("runtime.jdk.bundled", False))
+        self.template_renderer = template_renderer
 
     @property
     def download_url(self):
         # team repo
-        default_key = "{}_url".format(self.name)
+        if self.runtime_jdk_bundled:
+            default_key = "jdk.bundled.{}_url".format(self.name)
+        else:
+            default_key = "jdk.unbundled.{}_url".format(self.name)
         # rally.ini
         override_key = "{}.url".format(self.name)
         return self._url_for(override_key, default_key)
@@ -520,19 +564,7 @@ def _url_for(self, user_defined_key, default_key, mandatory=True):
                 raise exceptions.SystemSetupError("Neither config key [{}] nor [{}] is defined.".format(user_defined_key, default_key))
             else:
                 return None
-        return self._substitute_vars(url_template)
-
-    def _substitute_vars(self, s):
-        substitutions = {
-            "{{VERSION}}": self.version,
-            "{{OSNAME}}": sysstats.os_name().lower(),
-            "{{ARCH}}": sysstats.cpu_arch().lower()
-        }
-        r = s
-        for key, replacement in substitutions.items():
-            r = r.replace(key, replacement)
-        return r
-
+        return self.template_renderer.render(url_template)
 
     @property
     def cache(self):

esrally/rally.py

@@ -35,9 +35,18 @@ def create_arg_parser():
     def positive_number(v):
         value = int(v)
         if value <= 0:
-            raise argparse.ArgumentTypeError("must be positive but was %s" % value)
+            raise argparse.ArgumentTypeError("must be positive but was {}".format(value))
         return value
 
+    def runtime_jdk(v):
+        if v == "bundled":
+            return v
+        else:
+            try:
+                return positive_number(v)
+            except argparse.ArgumentTypeError:
+                raise argparse.ArgumentTypeError("must be a positive number or 'bundled' but was {}".format(v))
+
     # try to preload configurable defaults, but this does not work together with `--configuration-name` (which is undocumented anyway)
     cfg = config.Config()
     if cfg.config_present():
@@ -187,6 +196,9 @@ def positive_number(v):
         "--team-repository",
         help="Define the repository from where Rally will load teams and cars (default: default).",
         default="default")
+    download_parser.add_argument(
+        "--team-path",
+        help="Define the path to the car and plugin configurations to use.")
     download_parser.add_argument(
         "--distribution-version",
         help="Define the version of the Elasticsearch distribution to download. "
@@ -205,6 +217,14 @@ def positive_number(v):
         help="Define a comma-separated list of key:value pairs that are injected verbatim as variables for the car.",
         default=""
     )
+    download_parser.add_argument(
+        "--target-os",
+        help="The name of the target operating system for which an artifact should be downloaded (default: current OS)",
+    )
+    download_parser.add_argument(
+        "--target-arch",
+        help="The name of the CPU architecture for which an artifact should be downloaded (default: current architecture)",
+    )
 
     install_parser = subparsers.add_parser("install", help="Installs an Elasticsearch node locally")
     install_parser.add_argument(
@@ -303,7 +323,7 @@ def positive_number(v):
         default="")
     start_parser.add_argument(
         "--runtime-jdk",
-        type=positive_number,
+        type=runtime_jdk,
         help="The major version of the runtime JDK to use.",
         default=None)
     start_parser.add_argument(
@@ -338,7 +358,7 @@ def positive_number(v):
             default="")
         p.add_argument(
             "--runtime-jdk",
-            type=positive_number,
+            type=runtime_jdk,
             help="The major version of the runtime JDK to use.",
             default=None)
 
@@ -850,6 +870,9 @@ def main():
             console.println("--target-hosts and --client-options must define the same keys for multi cluster setups.")
             exit(1)
     # split by component?
+    if sub_command == "download":
+        cfg.add(config.Scope.applicationOverride, "mechanic", "target.os", args.target_os)
+        cfg.add(config.Scope.applicationOverride, "mechanic", "target.arch", args.target_arch)
     if sub_command == "list":
         cfg.add(config.Scope.applicationOverride, "system", "list.config.option", args.configuration)
         cfg.add(config.Scope.applicationOverride, "system", "list.races.max_results", args.limit)