Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow configuring Elasticsearch cluster name #1478

Merged
merged 11 commits into from
Jun 13, 2022
Merged

Allow configuring Elasticsearch cluster name #1478

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
07ca698
Allow configuring Elasticsearch cluster name
dnhatn Apr 17, 2022
198618e
Fix Docker cluster name
dnhatn Apr 20, 2022
30d4e50
Fix tests
dnhatn Apr 20, 2022
daa41b9
Lint
dnhatn Apr 20, 2022
7d3d35f
Add parameter to `race` sub command
dnhatn May 2, 2022
bfad99a
fix docs
dnhatn May 2, 2022
5127ed3
Merge remote-tracking branch 'elastic/master' into cluster_name
dnhatn Jun 13, 2022
0f84fb9
Add cluster-name opts to install and race parsers
dnhatn Jun 13, 2022
b630a50
Keep old docs
dnhatn Jun 13, 2022
6946dd0
Add an example with two clusters
dnhatn Jun 13, 2022
538418d
typo
dnhatn Jun 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions docs/command_line_reference.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,26 @@ A unique identifier for an installation. This id is automatically generated by R

Used to specify the current node's name in the cluster when it is setup via the ``install`` subcommand.

``cluster-name``
~~~~~~~~~~~~~~~~

.. warning::

This command line parameter is experimental. Expect the functionality and the command line interface to change significantly even in patch releases.

This parameter is useful in benchmarks involved multiple Elasticsearch clusters. It's used to configure the cluster name of the current Elasticsearch node when it is setup via the ``install`` or ``race`` subcommand. The following example sets up two Elasticsearch clusters: ``cluster-1`` and ``cluster-2``, and each has two nodes::

# install node-1 in cluster-1
esrally install --quiet --distribution-version=8.2.2 --node-name="node-1" --cluster-name=cluster-1 --network-host="192.168.1.1" --http-port=39200 --master-nodes="node-1" --seed-hosts="192.168.1.1:39300,192.168.1.2:39300"
# install node-2 in cluster-1
esrally install --quiet --distribution-version=8.2.2 --node-name="node-2" --cluster-name=cluster-1 --network-host="192.168.1.2" --http-port=39200 --master-nodes="node-1" --seed-hosts="192.168.1.1:39300,192.168.1.2:39300"
# install node-3 in cluster-2
esrally install --quiet --distribution-version=8.2.2 --node-name="node-3" --cluster-name=cluster-2 --network-host="192.168.1.3" --http-port=39200 --master-nodes="node-3" --seed-hosts="192.168.1.3:39300,192.168.1.4:39300"
# install node-4 in cluster-2
esrally install --quiet --distribution-version=8.2.2 --node-name="node-4" --cluster-name=cluster-2 --network-host="192.168.1.4" --http-port=39200 --master-nodes="node-3" --seed-hosts="192.168.1.3:39300,192.168.1.4:39300"

If the ``cluster-name`` parameter is not specified, Rally will use ``rally-benchmark`` as the default cluster name.

``network-host``
~~~~~~~~~~~~~~~~

Expand Down
17 changes: 12 additions & 5 deletions esrally/mechanic/provisioner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,26 +31,30 @@

def local(cfg, car, plugins, ip, http_port, all_node_ips, all_node_names, target_root, node_name):
distribution_version = cfg.opts("mechanic", "distribution.version", mandatory=False)
cluster_name = cfg.opts("mechanic", "cluster.name")

node_root_dir = os.path.join(target_root, node_name)

runtime_jdk_bundled = convert.to_bool(car.mandatory_var("runtime.jdk.bundled"))
runtime_jdk = car.mandatory_var("runtime.jdk")
_, java_home = java_resolver.java_home(runtime_jdk, cfg.opts("mechanic", "runtime.jdk"), runtime_jdk_bundled)

es_installer = ElasticsearchInstaller(car, java_home, node_name, node_root_dir, all_node_ips, all_node_names, ip, http_port)
es_installer = ElasticsearchInstaller(
car, java_home, node_name, cluster_name, node_root_dir, all_node_ips, all_node_names, ip, http_port
)
plugin_installers = [PluginInstaller(plugin, java_home) for plugin in plugins]

return BareProvisioner(es_installer, plugin_installers, distribution_version=distribution_version)


def docker(cfg, car, ip, http_port, target_root, node_name):
distribution_version = cfg.opts("mechanic", "distribution.version", mandatory=False)
cluster_name = cfg.opts("mechanic", "cluster.name")
rally_root = cfg.opts("node", "rally.root")

node_root_dir = os.path.join(target_root, node_name)

return DockerProvisioner(car, node_name, ip, http_port, node_root_dir, distribution_version, rally_root)
return DockerProvisioner(car, node_name, cluster_name, ip, http_port, node_root_dir, distribution_version, rally_root)


class NodeConfiguration:
Expand Down Expand Up @@ -250,6 +254,7 @@ def __init__(
car,
java_home,
node_name,
cluster_name,
node_root_dir,
all_node_ips,
all_node_names,
Expand All @@ -260,6 +265,7 @@ def __init__(
self.car = car
self.java_home = java_home
self.node_name = node_name
self.cluster_name = cluster_name
self.node_root_dir = node_root_dir
self.install_dir = os.path.join(node_root_dir, "install")
self.node_log_dir = os.path.join(node_root_dir, "logs", "server")
Expand Down Expand Up @@ -303,7 +309,7 @@ def variables(self):
network_host = self.node_ip

defaults = {
"cluster_name": "rally-benchmark",
"cluster_name": self.cluster_name,
"node_name": self.node_name,
"data_paths": self.data_paths,
"log_path": self.node_log_dir,
Expand Down Expand Up @@ -404,9 +410,10 @@ def sub_plugin_name(self):


class DockerProvisioner:
def __init__(self, car, node_name, ip, http_port, node_root_dir, distribution_version, rally_root):
def __init__(self, car, node_name, cluster_name, ip, http_port, node_root_dir, distribution_version, rally_root):
self.car = car
self.node_name = node_name
self.cluster_name = cluster_name
self.node_ip = ip
self.http_port = http_port
self.node_root_dir = node_root_dir
Expand All @@ -420,7 +427,7 @@ def __init__(self, car, node_name, ip, http_port, node_root_dir, distribution_ve
self.logger = logging.getLogger(__name__)

provisioner_defaults = {
"cluster_name": "rally-benchmark",
"cluster_name": self.cluster_name,
"node_name": self.node_name,
# we bind-mount the directories below on the host to these ones.
"install_root_path": "/usr/share/elasticsearch",
Expand Down
8 changes: 8 additions & 0 deletions esrally/rally.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,12 @@ def add_track_source(subparser):
help="A comma-separated list of the initial seed host IPs",
default="",
)
for p in [race_parser, install_parser]:
p.add_argument(
"--cluster-name",
help="The name of this Elasticsearch cluster",
default="rally-benchmark",
)

start_parser = subparsers.add_parser("start", help="Starts an Elasticsearch node locally")
start_parser.add_argument(
Expand Down Expand Up @@ -927,6 +933,7 @@ def dispatch_sub_command(arg_parser, args, cfg):
cfg.add(config.Scope.applicationOverride, "mechanic", "build.type", args.build_type)
cfg.add(config.Scope.applicationOverride, "mechanic", "runtime.jdk", args.runtime_jdk)
cfg.add(config.Scope.applicationOverride, "mechanic", "node.name", args.node_name)
cfg.add(config.Scope.applicationOverride, "mechanic", "cluster.name", args.cluster_name)
cfg.add(config.Scope.applicationOverride, "mechanic", "master.nodes", opts.csv_to_list(args.master_nodes))
cfg.add(config.Scope.applicationOverride, "mechanic", "seed.hosts", opts.csv_to_list(args.seed_hosts))
cfg.add(config.Scope.applicationOverride, "mechanic", "car.plugins", opts.csv_to_list(args.elasticsearch_plugins))
Expand Down Expand Up @@ -970,6 +977,7 @@ def dispatch_sub_command(arg_parser, args, cfg):
cfg.add(config.Scope.applicationOverride, "mechanic", "plugin.params", opts.to_dict(args.plugin_params))
cfg.add(config.Scope.applicationOverride, "mechanic", "preserve.install", convert.to_bool(args.preserve_install))
cfg.add(config.Scope.applicationOverride, "mechanic", "skip.rest.api.check", convert.to_bool(args.skip_rest_api_check))
cfg.add(config.Scope.applicationOverride, "mechanic", "cluster.name", args.cluster_name)

configure_reporting_params(args, cfg)
race(cfg, args.kill_running_processes)
Expand Down
3 changes: 2 additions & 1 deletion it/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ def install(self, distribution_version, node_name, car, http_port):
output = process.run_subprocess_with_output(
"esrally install --configuration-name={cfg} --quiet --distribution-version={dist} --build-type=tar "
"--http-port={http_port} --node={node_name} --master-nodes={node_name} --car={car} "
'--seed-hosts="127.0.0.1:{transport_port}"'.format(
'--seed-hosts="127.0.0.1:{transport_port}" --cluster-name={cfg}'.format(
cfg=self.cfg,
dist=distribution_version,
http_port=http_port,
Expand All @@ -179,6 +179,7 @@ def start(self, race_id):
raise AssertionError("Failed to start Elasticsearch test cluster.")
es = client.EsClientFactory(hosts=[{"host": "127.0.0.1", "port": self.http_port}], client_options={}).create()
client.wait_for_rest_layer(es)
assert es.info()["cluster_name"] == self.cfg

def stop(self):
if self.installation_id:
Expand Down
9 changes: 9 additions & 0 deletions tests/mechanic/provisioner_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ def null_apply_config(source_root_path, target_root_path, config_vars):
),
java_home="/usr/local/javas/java8",
node_name="rally-node-0",
cluster_name="rally-benchmark",
node_root_dir=HOME_DIR + "/.rally/benchmarks/races/unittest",
all_node_ips=["10.17.22.22", "10.17.22.23"],
all_node_names=["rally-node-0", "rally-node-1"],
Expand Down Expand Up @@ -157,6 +158,7 @@ def null_apply_config(source_root_path, target_root_path, config_vars):
),
java_home="/usr/local/javas/java8",
node_name="rally-node-0",
cluster_name="rally-benchmark",
node_root_dir=HOME_DIR + "/.rally/benchmarks/races/unittest",
all_node_ips=["10.17.22.22", "10.17.22.23"],
all_node_names=["rally-node-0", "rally-node-1"],
Expand Down Expand Up @@ -242,6 +244,7 @@ def null_apply_config(source_root_path, target_root_path, config_vars):
),
java_home="/usr/local/javas/java8",
node_name="rally-node-0",
cluster_name="rally-benchmark",
node_root_dir=HOME_DIR + "/.rally/benchmarks/races/unittest",
all_node_ips=["10.17.22.22", "10.17.22.23"],
all_node_names=["rally-node-0", "rally-node-1"],
Expand Down Expand Up @@ -322,6 +325,7 @@ def test_prepare_default_data_paths(self, mock_rm, mock_ensure_dir, mock_decompr
car=team.Car(names="defaults", root_path=None, config_paths="/tmp"),
java_home="/usr/local/javas/java8",
node_name="rally-node-0",
cluster_name="rally-benchmark",
all_node_ips=["10.17.22.22", "10.17.22.23"],
all_node_names=["rally-node-0", "rally-node-1"],
ip="10.17.22.23",
Expand Down Expand Up @@ -366,6 +370,7 @@ def test_prepare_user_provided_data_path(self, mock_rm, mock_ensure_dir, mock_de
),
java_home="/usr/local/javas/java8",
node_name="rally-node-0",
cluster_name="rally-benchmark",
all_node_ips=["10.17.22.22", "10.17.22.23"],
all_node_names=["rally-node-0", "rally-node-1"],
ip="10.17.22.23",
Expand Down Expand Up @@ -406,6 +411,7 @@ def test_invokes_hook_with_java_home(self):
),
java_home="/usr/local/javas/java8",
node_name="rally-node-0",
cluster_name="rally-benchmark",
all_node_ips=["10.17.22.22", "10.17.22.23"],
all_node_names=["rally-node-0", "rally-node-1"],
ip="10.17.22.23",
Expand All @@ -432,6 +438,7 @@ def test_invokes_hook_no_java_home(self):
),
java_home=None,
node_name="rally-node-0",
cluster_name="rally-benchmark",
all_node_ips=["10.17.22.22", "10.17.22.23"],
all_node_names=["rally-node-0", "rally-node-1"],
ip="10.17.22.23",
Expand Down Expand Up @@ -618,6 +625,7 @@ def test_provisioning_with_defaults(self, uuid4):
docker = provisioner.DockerProvisioner(
car=c,
node_name="rally-node-0",
cluster_name="rally-benchmark",
ip=node_ip,
http_port=39200,
node_root_dir=node_root_dir,
Expand Down Expand Up @@ -711,6 +719,7 @@ def test_provisioning_with_variables(self, uuid4):
docker = provisioner.DockerProvisioner(
car=c,
node_name="rally-node-0",
cluster_name="rally-benchmark",
ip=node_ip,
http_port=39200,
node_root_dir=node_root_dir,
Expand Down