From 9625c8ae01fa8e895eebb6478208b96ec181706a Mon Sep 17 00:00:00 2001 From: AdrianoDee Date: Mon, 9 Sep 2024 16:46:43 +0200 Subject: [PATCH 1/4] Update input model --- core/model/relval_step.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/model/relval_step.py b/core/model/relval_step.py index 4492f13..39d87e4 100644 --- a/core/model/relval_step.py +++ b/core/model/relval_step.py @@ -66,6 +66,7 @@ class RelValStep(ModelBase): 'lumisection': {}, 'run': [], 'label': '', + 'events': '', }, # Keeping output of this task 'keep_output': True, @@ -92,7 +93,9 @@ class RelValStep(ModelBase): }, '_input': { 'dataset': lambda ds: not ds or ModelBase.lambda_check('dataset')(ds), - 'label': lambda l: not l or ModelBase.lambda_check('label')(l) + 'label': lambda l: not l or ModelBase.lambda_check('label')(l), + 'events': lambda m: m == '' or int(m) > 0, + }, }, 'lumis_per_job': lambda l: l == '' or int(l) > 0, 'name': lambda n: ModelBase.matches_regex(n, '[a-zA-Z0-9_\\-]{1,150}'), From 7c51016782110638345eae66d344134643bd044d Mon Sep 17 00:00:00 2001 From: AdrianoDee Date: Sat, 7 Sep 2024 16:49:18 +0200 Subject: [PATCH 2/4] Implement event skim --- core/model/relval_step.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/core/model/relval_step.py b/core/model/relval_step.py index 39d87e4..fc0f4b1 100644 --- a/core/model/relval_step.py +++ b/core/model/relval_step.py @@ -280,7 +280,25 @@ def __build_das_command(self, step_index): command += 'dasgoclient --limit 0 ' command += f'--query "file dataset={dataset} run in [{run_chunk}]" ' command += f'>> {files_name}\n' + return (comment + '\n' + command).strip() + + events = input_dict['events'] + ## N.B. das-up-to-nevents.py exists only from 14_1_0_pre7 + cmssw_components = lambda x: x.strip().split("_") + cmssw_release = cmssw_components(self.get_release()) + cmssw_release >= cmssw_components("14_1_0_pre7") + + if events and check_das_up_to_nevents: + self.logger.info('Making a DAS command for step %s with max events', step_index) + files_name = f'step{step_index + 1}_files.txt' + comment = f'# Arguments for step {step_index + 1}:\n' + command = f'# Command for step {step_index + 1}:\n' + comment += f'# dataset: {dataset}\n' + comment += f'# events : {events}\n' + command += f'echo "" > {files_name}\n' + commnad += f'das-up-to-nevents.py -d {dataset} -e {events} --pc' + command += f'>> {files_name}\n' return (comment + '\n' + command).strip() return f'# Step {step_index + 1} is input dataset for next step: {dataset}' From 9bc0bfd15cf7b4298ffac86ef488118c3fcf367e Mon Sep 17 00:00:00 2001 From: adiflori Date: Thu, 17 Oct 2024 19:02:10 +0200 Subject: [PATCH 3/4] Fixes for input events --- core/controller/ticket_controller.py | 2 +- core/model/relval_step.py | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/core/controller/ticket_controller.py b/core/controller/ticket_controller.py index 5dfbe61..7e45d71 100644 --- a/core/controller/ticket_controller.py +++ b/core/controller/ticket_controller.py @@ -255,6 +255,7 @@ def recycle_input(self, relvals, relval_controller, recycle_input_of): "lumisection": {}, "run": [], "label": "", + "events": 0, } input_step_json["name"] += "_Recycled" input_step = RelValStep(input_step_json, relval, False) @@ -268,7 +269,6 @@ def make_relval_step(self, step_dict): """ # Deal with input file part input_dict = step_dict.get("input", {}) - input_dict.pop("events", None) # Deal with driver part arguments = step_dict.get("arguments", {}) diff --git a/core/model/relval_step.py b/core/model/relval_step.py index fc0f4b1..83c13b5 100644 --- a/core/model/relval_step.py +++ b/core/model/relval_step.py @@ -66,7 +66,7 @@ class RelValStep(ModelBase): 'lumisection': {}, 'run': [], 'label': '', - 'events': '', + 'events': 0, }, # Keeping output of this task 'keep_output': True, @@ -96,7 +96,6 @@ class RelValStep(ModelBase): 'label': lambda l: not l or ModelBase.lambda_check('label')(l), 'events': lambda m: m == '' or int(m) > 0, }, - }, 'lumis_per_job': lambda l: l == '' or int(l) > 0, 'name': lambda n: ModelBase.matches_regex(n, '[a-zA-Z0-9_\\-]{1,150}'), 'scram_arch': lambda s: not s or ModelBase.lambda_check('scram_arch')(s), @@ -112,12 +111,14 @@ def __init__(self, json_input=None, parent=None, check_attributes=True): json_input['gpu'] = schema.get('gpu') json_input['gpu']['requires'] = 'forbidden' step_input = json_input['input'] + for key, default_value in schema['input'].items(): if key not in step_input: step_input[key] = default_value else: json_input['driver'] = {k.lstrip('-'): v for k, v in json_input['driver'].items()} json_input['input'] = schema.get('input') + if json_input.get('gpu', {}).get('requires') not in ('optional', 'required'): json_input['gpu'] = schema.get('gpu') json_input['gpu']['requires'] = 'forbidden' @@ -283,13 +284,13 @@ def __build_das_command(self, step_index): return (comment + '\n' + command).strip() events = input_dict['events'] - + ## N.B. das-up-to-nevents.py exists only from 14_1_0_pre7 cmssw_components = lambda x: x.strip().split("_") cmssw_release = cmssw_components(self.get_release()) - cmssw_release >= cmssw_components("14_1_0_pre7") - - if events and check_das_up_to_nevents: + check_das_up_to_nevents = cmssw_release >= cmssw_components("14_1_0_pre7") + + if events > 0 and check_das_up_to_nevents: self.logger.info('Making a DAS command for step %s with max events', step_index) files_name = f'step{step_index + 1}_files.txt' comment = f'# Arguments for step {step_index + 1}:\n' @@ -297,7 +298,7 @@ def __build_das_command(self, step_index): comment += f'# dataset: {dataset}\n' comment += f'# events : {events}\n' command += f'echo "" > {files_name}\n' - commnad += f'das-up-to-nevents.py -d {dataset} -e {events} --pc' + command += f'das-up-to-nevents.py -d {dataset} -e {events} ' command += f'>> {files_name}\n' return (comment + '\n' + command).strip() @@ -350,6 +351,7 @@ def get_command(self, custom_fragment=None, for_submission=False): previous_input = previous.get('input') previous_lumisection = previous_input['lumisection'] previous_run = previous_input['run'] + previous_events = previous_input['events'] if previous_lumisection: # If there are lumi ranges, add a file with them and list of files as input arguments_dict['filein'] = f'"filelist:step{index}_files.txt"' @@ -357,6 +359,9 @@ def get_command(self, custom_fragment=None, for_submission=False): elif previous_run: # If there is a run whitelist, add the file arguments_dict['filein'] = f'"filelist:step{index}_files.txt"' + elif previous_events: + # If there is a max number of events, add the file + arguments_dict['filein'] = f'"filelist:step{index}_files.txt"' else: # If there are no lumi ranges, use input file normally previous_dataset = previous_input['dataset'] From c01904ef5ca2e8c2b87be7c0b2a176e57a7aa17d Mon Sep 17 00:00:00 2001 From: Adriano Di Florio Date: Mon, 21 Oct 2024 17:46:04 +0200 Subject: [PATCH 4/4] Small fix to check for events to be a number MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Geovanny González-Rodríguez --- core/model/relval_step.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/model/relval_step.py b/core/model/relval_step.py index 83c13b5..e0ebc99 100644 --- a/core/model/relval_step.py +++ b/core/model/relval_step.py @@ -94,7 +94,7 @@ class RelValStep(ModelBase): '_input': { 'dataset': lambda ds: not ds or ModelBase.lambda_check('dataset')(ds), 'label': lambda l: not l or ModelBase.lambda_check('label')(l), - 'events': lambda m: m == '' or int(m) > 0, + 'events': lambda m: m == '' or (m.isnumeric() and int(m) > 0), }, 'lumis_per_job': lambda l: l == '' or int(l) > 0, 'name': lambda n: ModelBase.matches_regex(n, '[a-zA-Z0-9_\\-]{1,150}'),