Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cleanup Pylint singleton issues #845

Merged
merged 12 commits into from
Dec 18, 2019
2 changes: 1 addition & 1 deletion esrally/actor.py
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ def bootstrap_actor_system(try_join=False, prefer_local_only=False, local_ip=Non
coordinator_ip = None
local_ip = None
else:
if system_base != "multiprocTCPBase" and system_base != "multiprocUDPBase":
if system_base not in ("multiprocTCPBase", "multiprocUDPBase"):
raise exceptions.SystemSetupError("Rally requires a network-capable system base but got [%s]." % system_base)
if not coordinator_ip:
raise exceptions.SystemSetupError("coordinator IP is required")
Expand Down
1 change: 0 additions & 1 deletion esrally/driver/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -717,7 +717,6 @@ def scroll_query(self, es, params):
body = mandatory(params, "body", self)
sort = "_doc"
scroll = "10s"
size = size
doc_type = params.get("type")
params = request_params
if doc_type is not None:
Expand Down
20 changes: 8 additions & 12 deletions esrally/mechanic/supplier.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ def _supply_requirements(sources, distribution, build, plugins, revisions, distr
# * --pipeline=from-sources-skip-build --distribution-version=X.Y.Z where the plugin should not be built but ES should be
# a distributed version.
# * --distribution-version=X.Y.Z --revision="my-plugin:abcdef" where the plugin should be built from sources.
# pylint: disable=consider-using-ternary
plugin_needs_build = (sources and build) or distribution
# be a bit more lenient when checking for plugin revisions. This allows users to specify `--revision="current"` and
# rely on Rally to do the right thing.
Expand Down Expand Up @@ -179,18 +180,13 @@ def __init__(self, suppliers):

def __call__(self, *args, **kwargs):
binaries = {}
try:
for supplier in self.suppliers:
supplier.fetch()

for supplier in self.suppliers:
supplier.prepare()

for supplier in self.suppliers:
supplier.add(binaries)
return binaries
except BaseException:
raise
for supplier in self.suppliers:
supplier.fetch()
for supplier in self.suppliers:
supplier.prepare()
for supplier in self.suppliers:
supplier.add(binaries)
return binaries


class ElasticsearchSourceSupplier:
Expand Down
4 changes: 2 additions & 2 deletions esrally/telemetry.py
Original file line number Diff line number Diff line change
Expand Up @@ -393,8 +393,8 @@ def record(self):
self.record_stats_per_index(indices["index"], indices["shards"])
except KeyError:
self.logger.warning(
"The 'indices' key in {0} does not contain an 'index' or 'shards' key "
"Maybe the output format of the {0} endpoint has changed. Skipping.".format(ccr_stats_api_endpoint)
"The 'indices' key in %s does not contain an 'index' or 'shards' key "
"Maybe the output format of the %s endpoint has changed. Skipping.", ccr_stats_api_endpoint, ccr_stats_api_endpoint
)

def record_stats_per_index(self, name, stats):
Expand Down
48 changes: 24 additions & 24 deletions esrally/track/loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -414,7 +414,7 @@ def prepare_document_set(self, document_set, data_root):
if self.is_locally_available(doc_path) and \
self.has_expected_size(doc_path, document_set.uncompressed_size_in_bytes):
break
elif document_set.has_compressed_corpus() and \
if document_set.has_compressed_corpus() and \
self.is_locally_available(archive_path) and \
self.has_expected_size(archive_path, document_set.compressed_size_in_bytes):
self.decompress(archive_path, doc_path, document_set.uncompressed_size_in_bytes)
Expand Down Expand Up @@ -621,6 +621,9 @@ def relative_glob(start, f):


def filter_included_tasks(t, filters):
if not filters:
return t

logger = logging.getLogger(__name__)

def match(task, filters):
Expand All @@ -629,29 +632,26 @@ def match(task, filters):
return True
return False

if not filters:
return t
else:
# always include administrative tasks
complete_filters = [track.AdminTaskFilter()] + filters

for challenge in t.challenges:
# don't modify the schedule while iterating over it
tasks_to_remove = []
for task in challenge.schedule:
if not match(task, complete_filters):
tasks_to_remove.append(task)
else:
leafs_to_remove = []
for leaf_task in task:
if not match(leaf_task, complete_filters):
leafs_to_remove.append(leaf_task)
for leaf_task in leafs_to_remove:
logger.info("Removing sub-task [%s] from challenge [%s] due to task filter.", leaf_task, challenge)
task.remove_task(leaf_task)
for task in tasks_to_remove:
logger.info("Removing task [%s] from challenge [%s] due to task filter.", task, challenge)
challenge.remove_task(task)
# always include administrative tasks
complete_filters = [track.AdminTaskFilter()] + filters

for challenge in t.challenges:
# don't modify the schedule while iterating over it
tasks_to_remove = []
for task in challenge.schedule:
if not match(task, complete_filters):
tasks_to_remove.append(task)
else:
leafs_to_remove = []
for leaf_task in task:
if not match(leaf_task, complete_filters):
leafs_to_remove.append(leaf_task)
for leaf_task in leafs_to_remove:
logger.info("Removing sub-task [%s] from challenge [%s] due to task filter.", leaf_task, challenge)
task.remove_task(leaf_task)
for task in tasks_to_remove:
logger.info("Removing task [%s] from challenge [%s] due to task filter.", task, challenge)
challenge.remove_task(task)

return t

Expand Down
1 change: 1 addition & 0 deletions esrally/track/track.py
Original file line number Diff line number Diff line change
Expand Up @@ -447,6 +447,7 @@ class OperationType(Enum):
def admin_op(self):
return self.value > 1000

# pylint: disable=too-many-return-statements
@classmethod
def from_hyphenated_string(cls, v):
if v == "force-merge":
Expand Down
1 change: 1 addition & 0 deletions esrally/utils/console.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ class CmdLineProgressReporter:
:param printer: allow use of a different print method to assist with patching in unittests
"""

# pylint: disable=E0601
def __init__(self, width, plain_output=False, printer=print):
self._width = width
self._first_print = True
Expand Down
1 change: 1 addition & 0 deletions esrally/utils/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,7 @@ def _run(args, fallback=None, only_first_line=False):
return result_lines[0]
else:
return result_lines
# pylint: disable=W0702
except:
return fallback

Expand Down