diff --git a/docs/elasticsearch_plugins.rst b/docs/elasticsearch_plugins.rst index 5cf294514..357daf2f7 100644 --- a/docs/elasticsearch_plugins.rst +++ b/docs/elasticsearch_plugins.rst @@ -96,16 +96,19 @@ To make this work, you need to manually edit Rally's configuration file in ``~/. plugin.my-plugin.remote.repo.url = git@github.com:example-org/my-plugin.git plugin.my-plugin.src.subdir = elasticsearch-extra/my-plugin - plugin.my-plugin.build.task = :my-plugin:plugin:assemble + plugin.my-plugin.build.command = ./gradlew :my-plugin:plugin:assemble plugin.my-plugin.build.artifact.subdir = plugin/build/distributions Let's discuss these properties one by one: * ``plugin.my-plugin.remote.repo.url`` (optional): This is needed to let Rally checkout the source code of the plugin. If this is a private repo, credentials need to be setup properly. If the source code is already locally available you may not need to define this property. The remote's name is assumed to be "origin" and this is not configurable. Also, only git is supported as revision control system. * ``plugin.my-plugin.src.subdir`` (mandatory): This is the directory to which the plugin will be checked out relative to ``src.root.dir``. In order to allow to build the plugin alongside Elasticsearch, the plugin needs to reside in a subdirectory of ``elasticsearch-extra`` (see also the `Elasticsearch testing documentation `_. -* ``plugin.my-plugin.build.task`` (mandatory): The Gradle task to run in order to build the plugin artifact. Note that this command is run from the Elasticsearch source directory as Rally assumes that you want to build your plugin alongside Elasticsearch (otherwise, see the next section). +* ``plugin.my-plugin.build.command`` (mandatory): The full Gradle command to run in order to build the plugin artifact. Note that this command is run from the Elasticsearch source directory as Rally assumes that you want to build your plugin alongside Elasticsearch (otherwise, see the next section). * ``plugin.my-plugin.build.artifact.subdir`` (mandatory): This is the subdirectory relative to ``plugin.my-plugin.src.subdir`` in which the final plugin artifact is located. +.. warning:: + ``plugin.my-plugin.build.command`` has replaced ``plugin.my-plugin.build.task`` in earlier Rally versions. It now requires the *full* build command. + In order to run a benchmark with ``my-plugin``, you'd invoke Rally as follows: ``esrally --revision="elasticsearch:some-elasticsearch-revision,my-plugin:some-plugin-revision" --elasticsearch-plugins="my-plugin"`` where you need to replace ``some-elasticsearch-revision`` and ``some-plugin-revision`` with the appropriate :ref:`git revisions `. Adjust other command line parameters (like track or car) accordingly. In order for this to work, you need to ensure that: * All prerequisites for source builds are installed. @@ -122,14 +125,17 @@ To make this work, you need to manually edit Rally's configuration file in ``~/. plugin.my-plugin.remote.repo.url = git@github.com:example-org/my-plugin.git plugin.my-plugin.src.dir = /path/to/your/plugin/sources - plugin.my-plugin.build.task = :my-plugin:plugin:assemble + plugin.my-plugin.build.command = /usr/local/bin/gradle :my-plugin:plugin:assemble plugin.my-plugin.build.artifact.subdir = build/distributions +.. warning:: + ``plugin.my-plugin.build.command`` has replaced ``plugin.my-plugin.build.task`` in earlier Rally versions. It now requires the *full* build command. + Let's discuss these properties one by one: * ``plugin.my-plugin.remote.repo.url`` (optional): This is needed to let Rally checkout the source code of the plugin. If this is a private repo, credentials need to be setup properly. If the source code is already locally available you may not need to define this property. The remote's name is assumed to be "origin" and this is not configurable. Also, only git is supported as revision control system. * ``plugin.my-plugin.src.dir`` (mandatory): This is the absolute directory to which the source code will be checked out. -* ``plugin.my-plugin.build.task`` (mandatory): The Gradle task to run in order to build the plugin artifact. This command is run from the plugin project's root directory. +* ``plugin.my-plugin.build.command`` (mandatory): The full Gradle command to run in order to build the plugin artifact. This command is run from the plugin project's root directory. * ``plugin.my-plugin.build.artifact.subdir`` (mandatory): This is the subdirectory relative to ``plugin.my-plugin.src.dir`` in which the final plugin artifact is located. In order to run a benchmark with ``my-plugin``, you'd invoke Rally as follows: ``esrally --distribution-version="elasticsearch-version" --revision="my-plugin:some-plugin-revision" --elasticsearch-plugins="my-plugin"`` where you need to replace ``elasticsearch-version`` with the correct release (e.g. 6.0.0) and ``some-plugin-revision`` with the appropriate :ref:`git revisions `. Adjust other command line parameters (like track or car) accordingly. In order for this to work, you need to ensure that: @@ -238,4 +244,3 @@ For this to work you need ensure two things: 1. The plugin needs to be available for the version that you want to benchmark (5.5.0 in the example above). 2. Rally will choose the most appropriate branch in the team repository before starting the benchmark. In practice, this will most likely be branch "5" for this example. Therefore you need to ensure that your plugin configuration is also available on that branch. See the `README in the team repository `_ to learn how the versioning scheme works. - diff --git a/esrally/config.py b/esrally/config.py index 32348a676..6e05e3355 100644 --- a/esrally/config.py +++ b/esrally/config.py @@ -106,7 +106,7 @@ def auto_load_local_config(base_config, additional_sections=None, config_file_cl class Config: - CURRENT_CONFIG_VERSION = 14 + CURRENT_CONFIG_VERSION = 15 """ Config is the main entry point to retrieve and set benchmark properties. It provides multiple scopes to allow overriding of values on @@ -269,8 +269,7 @@ def __init__(self, i=input, sec_i=getpass.getpass, o=console.println): self.o = o self.prompter = None - def create_config(self, config_file, advanced_config=False, assume_defaults=False, use_gradle_wrapper=False, - java_home=None, runtime_java_home=None): + def create_config(self, config_file, advanced_config=False, assume_defaults=False, java_home=None, runtime_java_home=None): """ Either creates a new configuration file or overwrites an existing one. Will ask the user for input on configurable properties and writes them to the configuration file in ~/.rally/rally.ini. @@ -279,9 +278,10 @@ def create_config(self, config_file, advanced_config=False, assume_defaults=Fals :param advanced_config: Whether to ask for properties that are not necessary for everyday use (on a dev machine). Default: False. :param assume_defaults: If True, assume the user accepted all values for which defaults are provided. Mainly intended for automatic configuration in CI run. Default: False. - :param use_gradle_wrapper: If True, use the Gradle wrapper, otherwise use the system's Gradle version. Default: False. """ + benchmark_from_sources = True self.prompter = Prompter(self.i, self.sec_i, self.o, assume_defaults) + if advanced_config: self.o("Running advanced configuration. You can get additional help at:") self.o("") @@ -302,7 +302,6 @@ def create_config(self, config_file, advanced_config=False, assume_defaults=Fals # Autodetect settings self.o("* Autodetecting available third-party software") git_path = io.guess_install_location("git") - gradle_bin = "./gradlew" if use_gradle_wrapper else io.guess_install_location("gradle") java_8_home = runtime_java_home if runtime_java_home else io.guess_java_home(major_version=8) java_10_home = java_home if java_home else io.guess_java_home(major_version=10) @@ -316,67 +315,51 @@ def create_config(self, config_file, advanced_config=False, assume_defaults=Fals auto_detected_java_home = None self.print_detection_result("git ", git_path) - self.print_detection_result("gradle ", gradle_bin) self.print_detection_result("JDK ", auto_detected_java_home, warn_if_missing=True, additional_message="You cannot benchmark Elasticsearch on this machine without a JDK.") self.o("") - # users that don't have Gradle available cannot benchmark from sources - benchmark_from_sources = gradle_bin - - if not benchmark_from_sources: - self.o("********************************************************************************") - self.o("You don't have the required software to benchmark Elasticsearch source builds.") - self.o("") - self.o("You can still benchmark binary distributions with e.g.:") - self.o("") - self.o(" %s --distribution-version=6.0.0" % PROGRAM_NAME) - self.o("********************************************************************************") - self.o("") - root_dir = io.normalize_path(os.path.abspath(os.path.join(config_file.config_dir, "benchmarks"))) if advanced_config: root_dir = io.normalize_path(self._ask_property("Enter the benchmark data directory", default_value=root_dir)) else: self.o("* Setting up benchmark data directory in %s" % root_dir) - if benchmark_from_sources: - if not java_10_home or jvm.is_early_access_release(java_10_home): - raw_java_10_home = self._ask_property("Enter the JDK 10 root directory", check_path_exists=True, mandatory=False) - if raw_java_10_home and jvm.major_version(raw_java_10_home) == 10 and not jvm.is_early_access_release(raw_java_10_home): - java_10_home = io.normalize_path(raw_java_10_home) if raw_java_10_home else None - else: - benchmark_from_sources = False - self.o("********************************************************************************") - self.o("You don't have a valid JDK 10 installation and cannot benchmark source builds.") - self.o("") - self.o("You can still benchmark binary distributions with e.g.:") - self.o("") - self.o(" %s --distribution-version=6.0.0" % PROGRAM_NAME) - self.o("********************************************************************************") - self.o("") - - if benchmark_from_sources: - # We try to autodetect an existing ES source directory - guess = self._guess_es_src_dir() - if guess: - source_dir = guess - logger.debug("Autodetected Elasticsearch project directory at [%s]." % source_dir) + if not java_10_home or jvm.is_early_access_release(java_10_home): + raw_java_10_home = self._ask_property("Enter the JDK 10 root directory", check_path_exists=True, mandatory=False) + if raw_java_10_home and jvm.major_version(raw_java_10_home) == 10 and not jvm.is_early_access_release(raw_java_10_home): + java_10_home = io.normalize_path(raw_java_10_home) if raw_java_10_home else None else: - default_src_dir = os.path.join(root_dir, "src", "elasticsearch") - logger.debug("Could not autodetect Elasticsearch project directory. Providing [%s] as default." % default_src_dir) - source_dir = default_src_dir - - if advanced_config: - source_dir = io.normalize_path(self._ask_property("Enter your Elasticsearch project directory:", - default_value=source_dir)) - if not advanced_config: - self.o("* Setting up benchmark source directory in %s" % source_dir) + benchmark_from_sources = False + self.o("********************************************************************************") + self.o("You don't have a valid JDK 10 installation and cannot benchmark source builds.") + self.o("") + self.o("You can still benchmark binary distributions with e.g.:") self.o("") + self.o(" %s --distribution-version=6.0.0" % PROGRAM_NAME) + self.o("********************************************************************************") + self.o("") + + # We try to autodetect an existing ES source directory + guess = self._guess_es_src_dir() + if guess: + source_dir = guess + logger.debug("Autodetected Elasticsearch project directory at [%s]." % source_dir) + else: + default_src_dir = os.path.join(root_dir, "src", "elasticsearch") + logger.debug("Could not autodetect Elasticsearch project directory. Providing [%s] as default." % default_src_dir) + source_dir = default_src_dir + + if advanced_config: + source_dir = io.normalize_path(self._ask_property("Enter your Elasticsearch project directory:", + default_value=source_dir)) + if not advanced_config: + self.o("* Setting up benchmark source directory in %s" % source_dir) + self.o("") - # Not everybody might have SSH access. Play safe with the default. It may be slower but this will work for everybody. - repo_url = "https://github.com/elastic/elasticsearch.git" + # Not everybody might have SSH access. Play safe with the default. It may be slower but this will work for everybody. + repo_url = "https://github.com/elastic/elasticsearch.git" if auto_detected_java_home: java_home = auto_detected_java_home @@ -437,19 +420,13 @@ def create_config(self, config_file, advanced_config=False, assume_defaults=Fals config["node"] = {} config["node"]["root.dir"] = root_dir - if benchmark_from_sources: - # user has provided the Elasticsearch directory but the root for Elasticsearch and related plugins will be one level above - final_source_dir = io.normalize_path(os.path.abspath(os.path.join(source_dir, os.pardir))) - config["node"]["src.root.dir"] = final_source_dir - - config["source"] = {} - config["source"]["remote.repo.url"] = repo_url - # the Elasticsearch directory is just the last path component (relative to the source root directory) - config["source"]["elasticsearch.src.subdir"] = io.basename(source_dir) + final_source_dir = io.normalize_path(os.path.abspath(os.path.join(source_dir, os.pardir))) + config["node"]["src.root.dir"] = final_source_dir - if gradle_bin: - config["build"] = {} - config["build"]["gradle.bin"] = gradle_bin + config["source"] = {} + config["source"]["remote.repo.url"] = repo_url + # the Elasticsearch directory is just the last path component (relative to the source root directory) + config["source"]["elasticsearch.src.subdir"] = io.basename(source_dir) config["runtime"] = {} if java_home: @@ -749,7 +726,7 @@ def migrate(config_file, current_version, target_version, out=print, i=input): if "local.src.dir" in config["source"]: previous_root = config["source"].pop("local.src.dir") logger.info("Set [source][local.src.dir] to [%s]." % previous_root) - # if this directory was Rally's default location, then move it on the file system because to allow for checkouts of plugins + # if this directory was Rally's default location, then move it on the file system to allow for checkouts of plugins # in the sibling directory. if previous_root == os.path.join(config["node"]["root.dir"], "src"): new_root_dir_all_sources = previous_root @@ -867,6 +844,32 @@ def migrate(config_file, current_version, target_version, out=print, i=input): current_version = 14 config["meta"]["config.version"] = str(current_version) + if current_version == 14 and target_version > current_version: + # Be agnostic about build tools. Let use specify build commands for plugins and elasticsearch + # but also use gradlew by default for Elasticsearch and Core plugin builds, if nothing else has been specified. + + def migrate_source_plugin_build_tasks(config): + if 'source' not in config: + return + for k, v in config['source'].items(): + plugin_match = re.match('^plugin\.([^.]+)\.build\.task$',k) + if plugin_match != None and len(plugin_match.groups()) > 0 : + plugin_name = plugin_match.group(1) + new_key = "plugin.{}.build.command".format(plugin_name) + new_value = "./gradlew {}".format(v) + config['source'].pop(k) + config['source'][new_key] = new_value + logger.info("Migrating build.task for plugin [{}] from [{}] to [{}].".format(plugin_match.group(1), v, new_value)) + + logger.info("Migrating configuration version from 14 to 15.") + if "build" in config: + logger.info("Removing [build] section as part of the configuration migration. Rally now uses ./gradlew to build Elasticsearch.") + config.pop("build", None) + migrate_source_plugin_build_tasks(config) + + current_version = 15 + config["meta"]["config.version"] = str(current_version) + # all migrations done config_file.store(config) logger.info("Successfully self-upgraded configuration to version [%s]" % target_version) diff --git a/esrally/mechanic/supplier.py b/esrally/mechanic/supplier.py index b32106c43..bd075df8e 100644 --- a/esrally/mechanic/supplier.py +++ b/esrally/mechanic/supplier.py @@ -10,8 +10,9 @@ logger = logging.getLogger("rally.supplier") -CLEAN_TASK = "clean" -ASSEMBLE_TASK = ":distribution:archives:tar:assemble" +GRADLE_BINARY = "./gradlew" +CLEAN_COMMAND = "{} clean".format(GRADLE_BINARY) +ASSEMBLE_COMMAND = "{} :distribution:archives:tar:assemble".format(GRADLE_BINARY) # e.g. my-plugin:current - we cannot simply use String#split(":") as this would not work for timestamp-based revisions REVISION_PATTERN = r"(\w.*?):(.*)" @@ -26,10 +27,9 @@ def create(cfg, sources, distribution, build, challenge_root_path, plugins): suppliers = [] if build_needed: - gradle = cfg.opts("build", "gradle.bin") java10_home = _java10_home(cfg) es_src_dir = os.path.join(_src_dir(cfg), _config_value(src_config, "elasticsearch.src.subdir")) - builder = Builder(es_src_dir, gradle, java10_home, challenge_root_path) + builder = Builder(es_src_dir, java10_home, challenge_root_path) else: builder = None @@ -184,7 +184,7 @@ def fetch(self): def prepare(self): if self.builder: - self.builder.build([CLEAN_TASK, ASSEMBLE_TASK]) + self.builder.build([CLEAN_COMMAND, ASSEMBLE_COMMAND]) def add(self, binaries): binaries["elasticsearch"] = self.resolve_binary() @@ -230,8 +230,8 @@ def fetch(self): def prepare(self): if self.builder: - task = _config_value(self.src_config, "plugin.%s.build.task" % self.plugin.name) - self.builder.build([task], override_src_dir=self.override_build_dir) + command = _config_value(self.src_config, "plugin.{}.build.command".format(self.plugin.name)) + self.builder.build([command], override_src_dir=self.override_build_dir) def add(self, binaries): binaries[self.plugin.name] = self.resolve_binary() @@ -262,8 +262,8 @@ def fetch(self): def prepare(self): if self.builder: - task = ":plugins:%s:assemble" % self.plugin.name - self.builder.build([task]) + command = "{} :plugins:{}:assemble".format(GRADLE_BINARY, self.plugin.name) + self.builder.build([command]) def add(self, binaries): binaries[self.plugin.name] = self.resolve_binary() @@ -433,38 +433,37 @@ class Builder: It is not intended to be used directly but should be triggered by its mechanic. """ - def __init__(self, src_dir, gradle=None, java_home=None, log_dir=None): + def __init__(self, src_dir, java_home=None, log_dir=None): self.src_dir = src_dir - self.gradle = gradle self.java_home = java_home self.log_dir = log_dir - def build(self, tasks, override_src_dir=None): - for task in tasks: - self.run(task, override_src_dir) + def build(self, commands, override_src_dir=None): + for command in commands: + self.run(command, override_src_dir) - def run(self, task, override_src_dir=None): + def run(self, command, override_src_dir=None): from esrally.utils import jvm src_dir = self.src_dir if override_src_dir is None else override_src_dir logger.info("Building from sources in [%s]." % src_dir) - logger.info("Executing %s %s..." % (self.gradle, task)) + logger.info("Executing %s..." % (command,)) io.ensure_dir(self.log_dir) log_file = "%s/build.log" % self.log_dir # we capture all output to a dedicated build log file - build_cmd = "export JAVA_HOME=%s; cd %s; %s %s >> %s 2>&1" % (self.java_home, src_dir, self.gradle, task, log_file) - logger.info("Running build command [%s]" % build_cmd) + build_cmd = "export JAVA_HOME={}; cd {}; {} >> {} 2>&1".format(self.java_home, src_dir, command, log_file) + logger.info("Running build command [{}]".format(build_cmd)) if process.run_subprocess(build_cmd): - msg = "Executing '%s %s' failed. The last 20 lines in the build log file are:\n" % (self.gradle, task) + msg = "Executing '{}' failed. The last 20 lines in the build log file are:\n".format(command) msg += "=========================================================================================================\n" with open(log_file, "r", encoding="utf-8") as f: msg += "\t" msg += "\t".join(f.readlines()[-20:]) msg += "=========================================================================================================\n" - msg += "The full build log is available at [%s]." % log_file + msg += "The full build log is available at [{}].".format(log_file) raise BuildError(msg) diff --git a/esrally/rally.py b/esrally/rally.py index 12b5b13a8..fd122c243 100644 --- a/esrally/rally.py +++ b/esrally/rally.py @@ -214,12 +214,6 @@ def positive_number(v): help="Automatically accept all options with default values (default: false)", default=False, action="store_true") - # TODO #412: Remove this option. Rally will then always use the Gradle Wrapper. - # undocumented - only as a workaround to ensure integration tests are always working, even if Gradle is not installed. - p.add_argument( - "--use-gradle-wrapper", - default=False, - action="store_true") # undocumented - only as a workaround for integration tests p.add_argument("--java-home", default=None) p.add_argument("--runtime-java-home", default=None) @@ -439,7 +433,6 @@ def ensure_configuration_present(cfg, args, sub_command): config.ConfigFactory().create_config(cfg.config_file, advanced_config=args.advanced_config, assume_defaults=args.assume_defaults, - use_gradle_wrapper=args.use_gradle_wrapper, java_home=args.java_home, runtime_java_home=args.runtime_java_home) exit(0) diff --git a/esrally/utils/io.py b/esrally/utils/io.py index b8ab0d19d..8d981fc9e 100644 --- a/esrally/utils/io.py +++ b/esrally/utils/io.py @@ -375,7 +375,7 @@ def guess_install_location(binary_name, fallback=None): """ Checks whether a given binary is available on the user's path. - :param binary_name: The name of the binary, e.g. tail, gradle, mvn. + :param binary_name: The name of the binary, e.g. tail, mvn. :param fallback: A fallback to return if the binary could not be found on the path. :return: The full path to the provided binary or the provided fallback. """ diff --git a/integration-test.sh b/integration-test.sh index 2fbc56256..7533d6e2f 100755 --- a/integration-test.sh +++ b/integration-test.sh @@ -72,14 +72,14 @@ function set_up { # if the build defines these variables we'll explicitly use them instead of auto-detection if [ -n "${JAVA_HOME}" ] && [ -n "${RUNTIME_JAVA_HOME}" ]; then # configure for tests with an in-memory metrics store - esrally configure --java-home="${JAVA_HOME}" --runtime-java-home="${RUNTIME_JAVA_HOME}" --use-gradle-wrapper --assume-defaults --configuration-name="integration-test" + esrally configure --java-home="${JAVA_HOME}" --runtime-java-home="${RUNTIME_JAVA_HOME}" --assume-defaults --configuration-name="integration-test" # configure for tests with an Elasticsearch metrics store - esrally configure --java-home="${JAVA_HOME}" --runtime-java-home="${RUNTIME_JAVA_HOME}" --use-gradle-wrapper --assume-defaults --configuration-name="es-integration-test" + esrally configure --java-home="${JAVA_HOME}" --runtime-java-home="${RUNTIME_JAVA_HOME}" --assume-defaults --configuration-name="es-integration-test" else # configure for tests with an in-memory metrics store - esrally configure --use-gradle-wrapper --assume-defaults --configuration-name="integration-test" + esrally configure --assume-defaults --configuration-name="integration-test" # configure for tests with an Elasticsearch metrics store - esrally configure --use-gradle-wrapper --assume-defaults --configuration-name="es-integration-test" + esrally configure --assume-defaults --configuration-name="es-integration-test" fi diff --git a/tests/config_test.py b/tests/config_test.py index 91fc9b3e2..ae5da3dc1 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -237,7 +237,7 @@ class ConfigFactoryTests(TestCase): @mock.patch("esrally.utils.io.guess_java_home") @mock.patch("esrally.utils.io.guess_install_location") def test_create_simple_config(self, guess_install_location, guess_java_home, is_ea_release, working_copy): - guess_install_location.side_effect = ["/tests/usr/bin/git", "/tests/usr/bin/gradle"] + guess_install_location.side_effect = ["/tests/usr/bin/git"] guess_java_home.return_value = "/tests/java10/home" is_ea_release.return_value = False # Rally checks in the parent and sibling directories whether there is an ES working copy. We don't want this detection logic @@ -256,7 +256,7 @@ def test_create_simple_config(self, guess_install_location, guess_java_home, is_ print("%s::%s: %s" % (section, k, v)) self.assertTrue("meta" in config_store.config) - self.assertEqual("14", config_store.config["meta"]["config.version"]) + self.assertEqual("15", config_store.config["meta"]["config.version"]) self.assertTrue("system" in config_store.config) self.assertEqual("local", config_store.config["system"]["env.name"]) @@ -269,9 +269,6 @@ def test_create_simple_config(self, guess_install_location, guess_java_home, is_ self.assertEqual("https://github.com/elastic/elasticsearch.git", config_store.config["source"]["remote.repo.url"]) self.assertEqual("elasticsearch", config_store.config["source"]["elasticsearch.src.subdir"]) - self.assertTrue("build" in config_store.config) - self.assertEqual("/tests/usr/bin/gradle", config_store.config["build"]["gradle.bin"]) - self.assertTrue("runtime" in config_store.config) self.assertEqual("/tests/java10/home", config_store.config["runtime"]["java.home"]) self.assertEqual("/tests/java10/home", config_store.config["runtime"]["java10.home"]) @@ -314,7 +311,7 @@ def test_create_simple_config(self, guess_install_location, guess_java_home, is_ @mock.patch("os.path.exists") def test_create_simple_config_no_java_detected(self, path_exists, normalize_path, guess_install_location, guess_java_home, major_jvm_version, jvm_is_early_access_release): - guess_install_location.side_effect = ["/tests/usr/bin/git", "/tests/usr/bin/gradle"] + guess_install_location.side_effect = ["/tests/usr/bin/git"] guess_java_home.return_value = None normalize_path.side_effect = ["/home/user/.rally/benchmarks", "/tests/java10/home", "/tests/java8/home", "/home/user/.rally/benchmarks/src"] @@ -334,7 +331,7 @@ def test_create_simple_config_no_java_detected(self, path_exists, normalize_path @mock.patch("esrally.utils.io.guess_java_home") @mock.patch("esrally.utils.io.guess_install_location") def test_create_simple_config_no_java_installed(self, guess_install_location, guess_java_home): - guess_install_location.side_effect = ["/tests/usr/bin/git", "/tests/usr/bin/gradle"] + guess_install_location.side_effect = ["/tests/usr/bin/git"] guess_java_home.return_value = None # the input is the question for the JDK home and the JDK 10 home directory - the user does not define one @@ -351,7 +348,7 @@ def test_create_simple_config_no_java_installed(self, guess_install_location, gu @mock.patch("esrally.utils.io.guess_java_home") @mock.patch("esrally.utils.io.guess_install_location") def test_create_advanced_config(self, guess_install_location, guess_java_home, is_ea_release): - guess_install_location.side_effect = ["/tests/usr/bin/git", "/tests/usr/bin/gradle"] + guess_install_location.side_effect = ["/tests/usr/bin/git"] guess_java_home.side_effect = ["/tests/java8/home", "/tests/java10/home"] is_ea_release.return_value = False @@ -381,14 +378,12 @@ def test_create_advanced_config(self, guess_install_location, guess_java_home, i self.assertIsNotNone(config_store.config) self.assertTrue("meta" in config_store.config) - self.assertEqual("14", config_store.config["meta"]["config.version"]) + self.assertEqual("15", config_store.config["meta"]["config.version"]) self.assertTrue("system" in config_store.config) self.assertEqual("unittest-env", config_store.config["system"]["env.name"]) self.assertTrue("node" in config_store.config) self.assertEqual("/var/data/rally", config_store.config["node"]["root.dir"]) self.assertTrue("source" in config_store.config) - self.assertTrue("build" in config_store.config) - self.assertEqual("/tests/usr/bin/gradle", config_store.config["build"]["gradle.bin"]) self.assertTrue("runtime" in config_store.config) self.assertEqual("/tests/java8/home", config_store.config["runtime"]["java.home"]) self.assertEqual("/tests/java10/home", config_store.config["runtime"]["java10.home"]) @@ -1040,8 +1035,77 @@ def test_migrate_from_13_to_14_with_gradle_and_jdk8_ask_user_enter_valid(self, m } config_file.store(sample_config) config.migrate(config_file, 13, 14, out=null_output, i=MockInput(inputs=["/usr/lib/java10"])) - self.assertTrue(config_file.backup_created) self.assertEqual("14", config_file.config["meta"]["config.version"]) self.assertEqual("/usr/lib/java8", config_file.config["runtime"]["java.home"]) self.assertEqual("/usr/lib/java10", config_file.config["runtime"]["java10.home"]) + + def test_migrate_from_14_to_15_without_gradle(self): + config_file = InMemoryConfigStore("test") + sample_config = { + "meta": { + "config.version": 14 + } + } + config_file.store(sample_config) + config.migrate(config_file, 14, 15, out=null_output) + + self.assertTrue(config_file.backup_created) + self.assertEqual("15", config_file.config["meta"]["config.version"]) + + def test_migrate_from_14_to_15_with_gradle(self): + config_file = InMemoryConfigStore("test") + sample_config = { + "meta": { + "config.version": 14 + }, + "build": { + "gradle.bin": "/usr/local/bin/gradle" + }, + "runtime": { + "java.home": "/usr/lib/java8", + "java10.home": "/usr/lib/java10" + } + } + config_file.store(sample_config) + config.migrate(config_file, 14, 15, out=null_output) + + self.assertTrue(config_file.backup_created) + self.assertEqual("15", config_file.config["meta"]["config.version"]) + self.assertNotIn("build", config_file.config) + + @mock.patch("esrally.utils.io.guess_java_home") + @mock.patch("esrally.utils.jvm.is_early_access_release") + @mock.patch("esrally.utils.jvm.major_version") + def test_migrate_from_14_to_15_with_source_plugin_definition(self, major_version, is_early_access_release, guess_java_home): + guess_java_home.return_value = None + is_early_access_release.return_value = False + major_version.return_value = 10 + + config_file = InMemoryConfigStore("test") + sample_config = { + "meta": { + "config.version": 14 + }, + "build": { + "gradle.bin": "/usr/local/bin/gradle" + }, + "runtime": { + "java.home": "/usr/lib/java10", + "java10.home": "/usr/lib/java10" + }, + "source": { + "plugin.x-pack.remote.repo.url": "git@github.com:elastic/x-pack-elasticsearch.git", + "plugin.x-pack.src.subdir": "elasticsearch-extra/x-pack-elasticsearch", + "plugin.x-pack.build.task": ":x-pack-elasticsearch:plugin:assemble", + "plugin.x-pack.build.artifact.subdir": "plugin/build/distributions" + } + } + config_file.store(sample_config) + config.migrate(config_file, 14, 15, out=null_output) + + self.assertTrue(config_file.backup_created) + self.assertEqual("15", config_file.config["meta"]["config.version"]) + self.assertNotIn("build", config_file.config) + self.assertNotIn("plugin.x-pack.build.task", config_file.config["source"]) + self.assertEqual("./gradlew :x-pack-elasticsearch:plugin:assemble", config_file.config["source"]["plugin.x-pack.build.command"]) diff --git a/tests/mechanic/supplier_test.py b/tests/mechanic/supplier_test.py index cb1845059..41d609fcc 100644 --- a/tests/mechanic/supplier_test.py +++ b/tests/mechanic/supplier_test.py @@ -120,14 +120,14 @@ def test_build_on_jdk_8(self, jvm_major_version, mock_run_subprocess): jvm_major_version.return_value = 8 mock_run_subprocess.return_value = False - b = supplier.Builder(src_dir="/src", gradle="/usr/local/gradle", java_home="/opt/jdk8", log_dir="logs") - b.build([supplier.CLEAN_TASK, supplier.ASSEMBLE_TASK]) + b = supplier.Builder(src_dir="/src", java_home="/opt/jdk8", log_dir="logs") + b.build([supplier.CLEAN_COMMAND, supplier.ASSEMBLE_COMMAND]) calls = [ # Actual call - mock.call("export JAVA_HOME=/opt/jdk8; cd /src; /usr/local/gradle clean >> logs/build.log 2>&1"), + mock.call("export JAVA_HOME=/opt/jdk8; cd /src; ./gradlew clean >> logs/build.log 2>&1"), # Return value check - mock.call("export JAVA_HOME=/opt/jdk8; cd /src; /usr/local/gradle :distribution:archives:tar:assemble >> logs/build.log 2>&1"), + mock.call("export JAVA_HOME=/opt/jdk8; cd /src; ./gradlew :distribution:archives:tar:assemble >> logs/build.log 2>&1"), ] mock_run_subprocess.assert_has_calls(calls) @@ -138,14 +138,14 @@ def test_build_on_jdk_10(self, jvm_major_version, mock_run_subprocess): jvm_major_version.return_value = 10 mock_run_subprocess.return_value = False - b = supplier.Builder(src_dir="/src", gradle="/usr/local/gradle", java_home="/opt/jdk10", log_dir="logs") - b.build([supplier.CLEAN_TASK, supplier.ASSEMBLE_TASK]) + b = supplier.Builder(src_dir="/src", java_home="/opt/jdk10", log_dir="logs") + b.build([supplier.CLEAN_COMMAND, supplier.ASSEMBLE_COMMAND]) calls = [ # Actual call - mock.call("export JAVA_HOME=/opt/jdk10; cd /src; /usr/local/gradle clean >> logs/build.log 2>&1"), + mock.call("export JAVA_HOME=/opt/jdk10; cd /src; ./gradlew clean >> logs/build.log 2>&1"), # Return value check - mock.call("export JAVA_HOME=/opt/jdk10; cd /src; /usr/local/gradle " + mock.call("export JAVA_HOME=/opt/jdk10; cd /src; ./gradlew " ":distribution:archives:tar:assemble >> logs/build.log 2>&1"), ] @@ -163,7 +163,7 @@ def test_build(self): es = supplier.ElasticsearchSourceSupplier(revision="abc", es_src_dir="/src", remote_url="", builder=builder) es.prepare() - builder.build.assert_called_once_with([supplier.CLEAN_TASK, supplier.ASSEMBLE_TASK]) + builder.build.assert_called_once_with([supplier.CLEAN_COMMAND, supplier.ASSEMBLE_COMMAND]) @mock.patch("glob.glob", lambda p: ["elasticsearch.tar.gz"]) def test_add_elasticsearch_binary(self): @@ -408,7 +408,6 @@ def test_create_suppliers_for_es_distribution_plugin_source_build(self): cfg.add(config.Scope.application, "runtime", "java10.home", "/usr/local/bin/java10/") cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally") cfg.add(config.Scope.application, "node", "src.root.dir", "/opt/rally/src") - cfg.add(config.Scope.application, "build", "gradle.bin", "/opt/gradle") cfg.add(config.Scope.application, "source", "elasticsearch.src.subdir", "elasticsearch") cfg.add(config.Scope.application, "source", "plugin.community-plugin.src.dir", "/home/user/Projects/community-plugin") @@ -439,7 +438,6 @@ def test_create_suppliers_for_es_and_plugin_source_build(self): cfg.add(config.Scope.application, "runtime", "java10.home", "/usr/local/bin/java10/") cfg.add(config.Scope.application, "node", "root.dir", "/opt/rally") cfg.add(config.Scope.application, "node", "src.root.dir", "/opt/rally/src") - cfg.add(config.Scope.application, "build", "gradle.bin", "/opt/gradle") cfg.add(config.Scope.application, "source", "elasticsearch.src.subdir", "elasticsearch") cfg.add(config.Scope.application, "source", "remote.repo.url", "https://github.com/elastic/elasticsearch.git") cfg.add(config.Scope.application, "source", "plugin.community-plugin.src.subdir", "elasticsearch-extra/community-plugin")