From 43b05e0eaee86206fef693948669ed66f5626ef4 Mon Sep 17 00:00:00 2001 From: Dennis Lawler Date: Mon, 16 Dec 2019 14:50:43 -0800 Subject: [PATCH 01/12] Disable pylint E0601 to clear up false positive --- esrally/utils/console.py | 1 + 1 file changed, 1 insertion(+) diff --git a/esrally/utils/console.py b/esrally/utils/console.py index 91362bbf2..6f4ddea1f 100644 --- a/esrally/utils/console.py +++ b/esrally/utils/console.py @@ -152,6 +152,7 @@ class CmdLineProgressReporter: :param printer: allow use of a different print method to assist with patching in unittests """ + # pylint: disable=E0601 def __init__(self, width, plain_output=False, printer=print): self._width = width self._first_print = True From 400f3e6457d1482affa9d5528ac6701aec24e45a Mon Sep 17 00:00:00 2001 From: Dennis Lawler Date: Mon, 16 Dec 2019 15:07:11 -0800 Subject: [PATCH 02/12] Fix pylint E0911 (too many return statements) --- esrally/track/track.py | 57 +++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/esrally/track/track.py b/esrally/track/track.py index b39cdc309..428f54a66 100644 --- a/esrally/track/track.py +++ b/esrally/track/track.py @@ -450,63 +450,64 @@ def admin_op(self): @classmethod def from_hyphenated_string(cls, v): if v == "force-merge": - return OperationType.ForceMerge + r = OperationType.ForceMerge elif v == "index-stats": - return OperationType.IndicesStats + r = OperationType.IndicesStats elif v == "node-stats": - return OperationType.NodesStats + r = OperationType.NodesStats elif v == "search": - return OperationType.Search + r = OperationType.Search elif v == "cluster-health": - return OperationType.ClusterHealth + r = OperationType.ClusterHealth elif v == "bulk": - return OperationType.Bulk + r = OperationType.Bulk elif v == "raw-request": - return OperationType.RawRequest + r = OperationType.RawRequest elif v == "put-pipeline": - return OperationType.PutPipeline + r = OperationType.PutPipeline elif v == "refresh": - return OperationType.Refresh + r = OperationType.Refresh elif v == "create-index": - return OperationType.CreateIndex + r = OperationType.CreateIndex elif v == "delete-index": - return OperationType.DeleteIndex + r = OperationType.DeleteIndex elif v == "create-index-template": - return OperationType.CreateIndexTemplate + r = OperationType.CreateIndexTemplate elif v == "delete-index-template": - return OperationType.DeleteIndexTemplate + r = OperationType.DeleteIndexTemplate elif v == "shrink-index": - return OperationType.ShrinkIndex + r = OperationType.ShrinkIndex elif v == "create-ml-datafeed": - return OperationType.CreateMlDatafeed + r = OperationType.CreateMlDatafeed elif v == "delete-ml-datafeed": - return OperationType.DeleteMlDatafeed + r = OperationType.DeleteMlDatafeed elif v == "start-ml-datafeed": - return OperationType.StartMlDatafeed + r = OperationType.StartMlDatafeed elif v == "stop-ml-datafeed": - return OperationType.StopMlDatafeed + r = OperationType.StopMlDatafeed elif v == "create-ml-job": - return OperationType.CreateMlJob + r = OperationType.CreateMlJob elif v == "delete-ml-job": - return OperationType.DeleteMlJob + r = OperationType.DeleteMlJob elif v == "open-ml-job": - return OperationType.OpenMlJob + r = OperationType.OpenMlJob elif v == "close-ml-job": - return OperationType.CloseMlJob + r = OperationType.CloseMlJob elif v == "sleep": - return OperationType.Sleep + r = OperationType.Sleep elif v == "delete-snapshot-repository": - return OperationType.DeleteSnapshotRepository + r = OperationType.DeleteSnapshotRepository elif v == "create-snapshot-repository": - return OperationType.CreateSnapshotRepository + r = OperationType.CreateSnapshotRepository elif v == "restore-snapshot": - return OperationType.RestoreSnapshot + r = OperationType.RestoreSnapshot elif v == "wait-for-recovery": - return OperationType.WaitForRecovery + r = OperationType.WaitForRecovery elif v == "put-settings": - return OperationType.PutSettings + r = OperationType.PutSettings else: raise KeyError("No enum value for [%s]" % v) + return r class TaskNameFilter: From e7671ac84211b4c3e199363c1a6cc37051368f67 Mon Sep 17 00:00:00 2001 From: Dennis Lawler Date: Mon, 16 Dec 2019 15:29:00 -0800 Subject: [PATCH 03/12] Fix R1702 / too many nested blocks. --- esrally/track/loader.py | 46 ++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/esrally/track/loader.py b/esrally/track/loader.py index 6a92c10fd..ed0f19de4 100644 --- a/esrally/track/loader.py +++ b/esrally/track/loader.py @@ -621,6 +621,9 @@ def relative_glob(start, f): def filter_included_tasks(t, filters): + if not filters: + return t + logger = logging.getLogger(__name__) def match(task, filters): @@ -629,29 +632,26 @@ def match(task, filters): return True return False - if not filters: - return t - else: - # always include administrative tasks - complete_filters = [track.AdminTaskFilter()] + filters - - for challenge in t.challenges: - # don't modify the schedule while iterating over it - tasks_to_remove = [] - for task in challenge.schedule: - if not match(task, complete_filters): - tasks_to_remove.append(task) - else: - leafs_to_remove = [] - for leaf_task in task: - if not match(leaf_task, complete_filters): - leafs_to_remove.append(leaf_task) - for leaf_task in leafs_to_remove: - logger.info("Removing sub-task [%s] from challenge [%s] due to task filter.", leaf_task, challenge) - task.remove_task(leaf_task) - for task in tasks_to_remove: - logger.info("Removing task [%s] from challenge [%s] due to task filter.", task, challenge) - challenge.remove_task(task) + # always include administrative tasks + complete_filters = [track.AdminTaskFilter()] + filters + + for challenge in t.challenges: + # don't modify the schedule while iterating over it + tasks_to_remove = [] + for task in challenge.schedule: + if not match(task, complete_filters): + tasks_to_remove.append(task) + else: + leafs_to_remove = [] + for leaf_task in task: + if not match(leaf_task, complete_filters): + leafs_to_remove.append(leaf_task) + for leaf_task in leafs_to_remove: + logger.info("Removing sub-task [%s] from challenge [%s] due to task filter.", leaf_task, challenge) + task.remove_task(leaf_task) + for task in tasks_to_remove: + logger.info("Removing task [%s] from challenge [%s] due to task filter.", task, challenge) + challenge.remove_task(task) return t From 38fd43d45e66cbe1a3fa83b79b7b50079328a830 Mon Sep 17 00:00:00 2001 From: Dennis Lawler Date: Mon, 16 Dec 2019 16:01:05 -0800 Subject: [PATCH 04/12] Pylint suggested style changes (R1723,R1714,R1706) --- esrally/actor.py | 2 +- esrally/mechanic/supplier.py | 2 +- esrally/track/loader.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/esrally/actor.py b/esrally/actor.py index 4302682e2..750c6435e 100644 --- a/esrally/actor.py +++ b/esrally/actor.py @@ -224,7 +224,7 @@ def bootstrap_actor_system(try_join=False, prefer_local_only=False, local_ip=Non coordinator_ip = None local_ip = None else: - if system_base != "multiprocTCPBase" and system_base != "multiprocUDPBase": + if system_base not in ('multiprocTCPBase', 'multiprocUDPBase'): raise exceptions.SystemSetupError("Rally requires a network-capable system base but got [%s]." % system_base) if not coordinator_ip: raise exceptions.SystemSetupError("coordinator IP is required") diff --git a/esrally/mechanic/supplier.py b/esrally/mechanic/supplier.py index 62642c3ff..71096af21 100644 --- a/esrally/mechanic/supplier.py +++ b/esrally/mechanic/supplier.py @@ -144,7 +144,7 @@ def _supply_requirements(sources, distribution, build, plugins, revisions, distr # * --pipeline=from-sources-skip-build --distribution-version=X.Y.Z where the plugin should not be built but ES should be # a distributed version. # * --distribution-version=X.Y.Z --revision="my-plugin:abcdef" where the plugin should be built from sources. - plugin_needs_build = (sources and build) or distribution + plugin_needs_build = build if sources else distribution # be a bit more lenient when checking for plugin revisions. This allows users to specify `--revision="current"` and # rely on Rally to do the right thing. try: diff --git a/esrally/track/loader.py b/esrally/track/loader.py index ed0f19de4..a913d43ba 100644 --- a/esrally/track/loader.py +++ b/esrally/track/loader.py @@ -414,7 +414,7 @@ def prepare_document_set(self, document_set, data_root): if self.is_locally_available(doc_path) and \ self.has_expected_size(doc_path, document_set.uncompressed_size_in_bytes): break - elif document_set.has_compressed_corpus() and \ + if document_set.has_compressed_corpus() and \ self.is_locally_available(archive_path) and \ self.has_expected_size(archive_path, document_set.compressed_size_in_bytes): self.decompress(archive_path, doc_path, document_set.uncompressed_size_in_bytes) From fe18654750790a2b396f71de139daceeb2abf151 Mon Sep 17 00:00:00 2001 From: Dennis Lawler Date: Mon, 16 Dec 2019 16:03:07 -0800 Subject: [PATCH 05/12] Pylint W0127: assigning variable to itself. --- esrally/driver/runner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/esrally/driver/runner.py b/esrally/driver/runner.py index 55bb58f1a..8423b36d1 100644 --- a/esrally/driver/runner.py +++ b/esrally/driver/runner.py @@ -717,7 +717,6 @@ def scroll_query(self, es, params): body = mandatory(params, "body", self) sort = "_doc" scroll = "10s" - size = size doc_type = params.get("type") params = request_params if doc_type is not None: From 8d6e1aa19d8a2af83bbe7da39a2bd23c56532aaf Mon Sep 17 00:00:00 2001 From: Dennis Lawler Date: Mon, 16 Dec 2019 16:11:20 -0800 Subject: [PATCH 06/12] Disable broad exception inspection (W0702) for Pylint too. --- esrally/utils/io.py | 1 + 1 file changed, 1 insertion(+) diff --git a/esrally/utils/io.py b/esrally/utils/io.py index e4b40cd85..e0b815b04 100644 --- a/esrally/utils/io.py +++ b/esrally/utils/io.py @@ -374,6 +374,7 @@ def _run(args, fallback=None, only_first_line=False): return result_lines[0] else: return result_lines + # pylint: disable=W0702 except: return fallback From d5e0ab42b8456c895364540f7659d9ffc1d4e0d6 Mon Sep 17 00:00:00 2001 From: Dennis Lawler Date: Mon, 16 Dec 2019 16:13:52 -0800 Subject: [PATCH 07/12] Don't immediately re-raise after except (W0706) --- esrally/mechanic/supplier.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/esrally/mechanic/supplier.py b/esrally/mechanic/supplier.py index 71096af21..154970f59 100644 --- a/esrally/mechanic/supplier.py +++ b/esrally/mechanic/supplier.py @@ -179,18 +179,13 @@ def __init__(self, suppliers): def __call__(self, *args, **kwargs): binaries = {} - try: - for supplier in self.suppliers: - supplier.fetch() - - for supplier in self.suppliers: - supplier.prepare() - - for supplier in self.suppliers: - supplier.add(binaries) - return binaries - except BaseException: - raise + for supplier in self.suppliers: + supplier.fetch() + for supplier in self.suppliers: + supplier.prepare() + for supplier in self.suppliers: + supplier.add(binaries) + return binaries class ElasticsearchSourceSupplier: From 5c1be0ff21d3c7e23acd7d3beecc935c0b8c5a92 Mon Sep 17 00:00:00 2001 From: Dennis Lawler Date: Mon, 16 Dec 2019 16:18:12 -0800 Subject: [PATCH 08/12] Use logging interpolation instead of string format (pylint W1202) --- esrally/telemetry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esrally/telemetry.py b/esrally/telemetry.py index 10a334d6d..ed2cc067b 100644 --- a/esrally/telemetry.py +++ b/esrally/telemetry.py @@ -393,8 +393,8 @@ def record(self): self.record_stats_per_index(indices["index"], indices["shards"]) except KeyError: self.logger.warning( - "The 'indices' key in {0} does not contain an 'index' or 'shards' key " - "Maybe the output format of the {0} endpoint has changed. Skipping.".format(ccr_stats_api_endpoint) + "The 'indices' key in %s does not contain an 'index' or 'shards' key " + "Maybe the output format of the %s endpoint has changed. Skipping.", ccr_stats_api_endpoint, ccr_stats_api_endpoint ) def record_stats_per_index(self, name, stats): From d4116f477bab18013f0b9285fdbc4c4d35b27f71 Mon Sep 17 00:00:00 2001 From: Dennis Lawler Date: Tue, 17 Dec 2019 07:41:39 -0800 Subject: [PATCH 09/12] Put back double-quotes. --- esrally/actor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esrally/actor.py b/esrally/actor.py index 750c6435e..9f1b445d5 100644 --- a/esrally/actor.py +++ b/esrally/actor.py @@ -224,7 +224,7 @@ def bootstrap_actor_system(try_join=False, prefer_local_only=False, local_ip=Non coordinator_ip = None local_ip = None else: - if system_base not in ('multiprocTCPBase', 'multiprocUDPBase'): + if system_base not in ("multiprocTCPBase", "multiprocUDPBase"): raise exceptions.SystemSetupError("Rally requires a network-capable system base but got [%s]." % system_base) if not coordinator_ip: raise exceptions.SystemSetupError("coordinator IP is required") From 70f21973c76402cbf1fe7b8f21b1a2b59e7912ea Mon Sep 17 00:00:00 2001 From: Dennis Lawler Date: Tue, 17 Dec 2019 07:43:14 -0800 Subject: [PATCH 10/12] Revert "Fix pylint E0911 (too many return statements)" This reverts commit 400f3e6457d1482affa9d5528ac6701aec24e45a. --- esrally/track/track.py | 57 +++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/esrally/track/track.py b/esrally/track/track.py index 428f54a66..b39cdc309 100644 --- a/esrally/track/track.py +++ b/esrally/track/track.py @@ -450,64 +450,63 @@ def admin_op(self): @classmethod def from_hyphenated_string(cls, v): if v == "force-merge": - r = OperationType.ForceMerge + return OperationType.ForceMerge elif v == "index-stats": - r = OperationType.IndicesStats + return OperationType.IndicesStats elif v == "node-stats": - r = OperationType.NodesStats + return OperationType.NodesStats elif v == "search": - r = OperationType.Search + return OperationType.Search elif v == "cluster-health": - r = OperationType.ClusterHealth + return OperationType.ClusterHealth elif v == "bulk": - r = OperationType.Bulk + return OperationType.Bulk elif v == "raw-request": - r = OperationType.RawRequest + return OperationType.RawRequest elif v == "put-pipeline": - r = OperationType.PutPipeline + return OperationType.PutPipeline elif v == "refresh": - r = OperationType.Refresh + return OperationType.Refresh elif v == "create-index": - r = OperationType.CreateIndex + return OperationType.CreateIndex elif v == "delete-index": - r = OperationType.DeleteIndex + return OperationType.DeleteIndex elif v == "create-index-template": - r = OperationType.CreateIndexTemplate + return OperationType.CreateIndexTemplate elif v == "delete-index-template": - r = OperationType.DeleteIndexTemplate + return OperationType.DeleteIndexTemplate elif v == "shrink-index": - r = OperationType.ShrinkIndex + return OperationType.ShrinkIndex elif v == "create-ml-datafeed": - r = OperationType.CreateMlDatafeed + return OperationType.CreateMlDatafeed elif v == "delete-ml-datafeed": - r = OperationType.DeleteMlDatafeed + return OperationType.DeleteMlDatafeed elif v == "start-ml-datafeed": - r = OperationType.StartMlDatafeed + return OperationType.StartMlDatafeed elif v == "stop-ml-datafeed": - r = OperationType.StopMlDatafeed + return OperationType.StopMlDatafeed elif v == "create-ml-job": - r = OperationType.CreateMlJob + return OperationType.CreateMlJob elif v == "delete-ml-job": - r = OperationType.DeleteMlJob + return OperationType.DeleteMlJob elif v == "open-ml-job": - r = OperationType.OpenMlJob + return OperationType.OpenMlJob elif v == "close-ml-job": - r = OperationType.CloseMlJob + return OperationType.CloseMlJob elif v == "sleep": - r = OperationType.Sleep + return OperationType.Sleep elif v == "delete-snapshot-repository": - r = OperationType.DeleteSnapshotRepository + return OperationType.DeleteSnapshotRepository elif v == "create-snapshot-repository": - r = OperationType.CreateSnapshotRepository + return OperationType.CreateSnapshotRepository elif v == "restore-snapshot": - r = OperationType.RestoreSnapshot + return OperationType.RestoreSnapshot elif v == "wait-for-recovery": - r = OperationType.WaitForRecovery + return OperationType.WaitForRecovery elif v == "put-settings": - r = OperationType.PutSettings + return OperationType.PutSettings else: raise KeyError("No enum value for [%s]" % v) - return r class TaskNameFilter: From dcec83e7cf942d995c21b043377ab1448ebefaf2 Mon Sep 17 00:00:00 2001 From: Dennis Lawler Date: Tue, 17 Dec 2019 08:11:14 -0800 Subject: [PATCH 11/12] Disable too-many-return-statements for from_hyphenated_string --- esrally/track/track.py | 1 + 1 file changed, 1 insertion(+) diff --git a/esrally/track/track.py b/esrally/track/track.py index b39cdc309..d64c5980d 100644 --- a/esrally/track/track.py +++ b/esrally/track/track.py @@ -447,6 +447,7 @@ class OperationType(Enum): def admin_op(self): return self.value > 1000 + # pylint: disable=too-many-return-statements @classmethod def from_hyphenated_string(cls, v): if v == "force-merge": From 6d98d1988f1469c776a422b25c7185daca4f75c4 Mon Sep 17 00:00:00 2001 From: Dennis Lawler Date: Tue, 17 Dec 2019 08:14:24 -0800 Subject: [PATCH 12/12] Revert to non-ternary condition, disable pylint warning. --- esrally/mechanic/supplier.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esrally/mechanic/supplier.py b/esrally/mechanic/supplier.py index 154970f59..06b89d760 100644 --- a/esrally/mechanic/supplier.py +++ b/esrally/mechanic/supplier.py @@ -144,7 +144,8 @@ def _supply_requirements(sources, distribution, build, plugins, revisions, distr # * --pipeline=from-sources-skip-build --distribution-version=X.Y.Z where the plugin should not be built but ES should be # a distributed version. # * --distribution-version=X.Y.Z --revision="my-plugin:abcdef" where the plugin should be built from sources. - plugin_needs_build = build if sources else distribution + # pylint: disable=consider-using-ternary + plugin_needs_build = (sources and build) or distribution # be a bit more lenient when checking for plugin revisions. This allows users to specify `--revision="current"` and # rely on Rally to do the right thing. try: