From 6cfc2b3c701f2d05e8d738cac8c47fa67b8f0de4 Mon Sep 17 00:00:00 2001 From: jean-roch Date: Wed, 20 Mar 2024 12:33:01 +0100 Subject: [PATCH 1/4] HCAL nano and HCALCalib nano in autoNano --- Configuration/Applications/python/ConfigBuilder.py | 9 +++++++-- DPGAnalysis/HcalNanoAOD/python/hcalNano_cff.py | 2 +- PhysicsTools/NanoAOD/python/autoNANO.py | 4 ++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Configuration/Applications/python/ConfigBuilder.py b/Configuration/Applications/python/ConfigBuilder.py index 549b373761edd..8925ed90462e0 100644 --- a/Configuration/Applications/python/ConfigBuilder.py +++ b/Configuration/Applications/python/ConfigBuilder.py @@ -1796,8 +1796,11 @@ def prepare_PATGEN(self, stepSpec = "miniGEN"): def prepare_NANO(self, stepSpec = '' ): print(f"in prepare_nano {stepSpec}") ''' Enrich the schedule with NANO ''' - _,_nanoSeq,_nanoCff = self.loadDefaultOrSpecifiedCFF(stepSpec,self.NANODefaultCFF,self.NANODefaultSeq) - + if not '@' in stepSpec: + _,_nanoSeq,_nanoCff = self.loadDefaultOrSpecifiedCFF(stepSpec,self.NANODefaultCFF,self.NANODefaultSeq) + else: + _nanoSeq = stepSpec + # create full specified sequence using autoNANO from PhysicsTools.NanoAOD.autoNANO import autoNANO, expandNanoMapping # if not a autoNANO mapping, load an empty customization, which later will be converted into the default. @@ -1818,9 +1821,11 @@ def prepare_NANO(self, stepSpec = '' ): for _subSeq in _nanoSeq: if '.' in _subSeq: _cff,_seq = _subSeq.split('.') + print("NANO: scheduling:",_seq,"from",_cff) self.loadAndRemember(_cff) _seqToSchedule.append(_seq) else: + print("NANO: scheduling:",_subSeq) _seqToSchedule.append(_subSeq) self.scheduleSequence('+'.join(_seqToSchedule), 'nanoAOD_step') diff --git a/DPGAnalysis/HcalNanoAOD/python/hcalNano_cff.py b/DPGAnalysis/HcalNanoAOD/python/hcalNano_cff.py index 6be527c15d59b..b9a83af36e977 100644 --- a/DPGAnalysis/HcalNanoAOD/python/hcalNano_cff.py +++ b/DPGAnalysis/HcalNanoAOD/python/hcalNano_cff.py @@ -72,7 +72,7 @@ def customiseHcalCalib(process): process.raw2digi_step = cms.Path(process.hcalCalibDigiSequence, process.RawToDigiTask) # Insert the HLT filter at start of user path and nanoaod endpath - process.user_step.insert(0, process.hcalCalibHLTFilter) + process.nanoAOD_step.insert(0, process.hcalCalibHLTFilter) process.NANOAODoutput_step.insert(0, process.hcalCalibHLTFilter) return process diff --git a/PhysicsTools/NanoAOD/python/autoNANO.py b/PhysicsTools/NanoAOD/python/autoNANO.py index b580e42007ec3..dd8bae68ee14f 100644 --- a/PhysicsTools/NanoAOD/python/autoNANO.py +++ b/PhysicsTools/NanoAOD/python/autoNANO.py @@ -37,6 +37,10 @@ def expandNanoMapping(seqList, mapping, key): 'customize': 'DPGAnalysis/MuonTools/muNtupleProducer_cff.muDPGNanoCustomize'}, 'MUDPGBKG' : {'sequence': 'DPGAnalysis/MuonTools/muNtupleProducerBkg_cff.muDPGNanoProducerBkg', 'customize': 'DPGAnalysis/MuonTools/muNtupleProducerBkg_cff.muDPGNanoBkgCustomize'}, + # HCAL favlours: + 'HCAL' : {'sequence': 'DPGAnalysis/HcalNanoAOD/hcalNano_cff.hcalNanoTask'}, + 'HCALCalib' : { 'sequence': 'DPGAnalysis/HcalNanoAOD/hcalNano_cff.hcalNanoTask', + 'customize': 'DPGAnalysis/HcalNanoAOD/hcalNano_cff.customiseHcalCalib'}, #EGM flavours: add variables through customize 'EGM' : {'sequence': '@PHYS', 'customize' : '@PHYS+PhysicsTools/NanoAOD/egamma_custom_cff.addExtraEGammaVarsCustomize'}, From 07a31ab87cc29f0350e7fe2863420e8b61f27422 Mon Sep 17 00:00:00 2001 From: jean-roch Date: Wed, 20 Mar 2024 12:34:34 +0100 Subject: [PATCH 2/4] hcal nano test to relval_nano matrix --- .../PyReleaseValidation/python/relval_nano.py | 36 ++++++++++++------- .../python/relval_production.py | 3 -- .../python/relval_steps.py | 20 ----------- 3 files changed, 24 insertions(+), 35 deletions(-) diff --git a/Configuration/PyReleaseValidation/python/relval_nano.py b/Configuration/PyReleaseValidation/python/relval_nano.py index 2f48c05b380cc..4e75130807508 100644 --- a/Configuration/PyReleaseValidation/python/relval_nano.py +++ b/Configuration/PyReleaseValidation/python/relval_nano.py @@ -145,6 +145,9 @@ def subnext(self): steps['ZeroBias2023DRAW13.0']={'INPUT':InputInfo(location='STD', ls={369978: [[1, 800]]}, dataSet='/ZeroBias/Run2023D-v1/RAW')} +steps['TestEnablesEcalHcal2023C']={'INPUT':InputInfo(location='STD', ls={368489: [[46,546]]}, + dataSet='/TestEnablesEcalHcal/Run2023C-Express-v4/RAW')} + steps['NANO_data13.0']=merge([{'--era':'Run3', '--conditions':'auto:run3_data'}, _NANO_data]) @@ -154,18 +157,24 @@ def subnext(self): steps['muDPGNANO_data13.0']=merge([{'-s' : 'RAW2DIGI,NANO:@MUDPG', - '--conditions':'auto:run3_data', - '-n' : '100', - '--era' : 'Run3', - '--datatier':'NANOAOD', - '--eventcontent':'NANOAOD'}]) + '-n' : '100',}, + steps['NANO_data13.0']]) + steps['muDPGNANOBkg_data13.0']=merge([{'-s' : 'RAW2DIGI,NANO:@MUDPGBKG', - '--conditions':'auto:run3_data', - '-n' : '100', - '--era' : 'Run3', - '--datatier':'NANOAOD', - '--eventcontent':'NANOAOD'}]) + '-n' : '100',}, + steps['NANO_data13.0']]) + + +steps['hcalDPGNANO_data13.0']=merge([{'-s' : 'RAW2DIGI,RECO,NANO:@HCAL', + '-n' : '100', + '--processName': 'NANO',}, + steps['NANO_data13.0']]) + +steps['hcalDPGCalibNANO_data13.0']=merge([{'-s' : 'RAW2DIGI,RECO,NANO:@HCALCalib', + '-n' : '100', + '--processName': 'NANO',}, + steps['NANO_data13.0']]) steps['muPOGNANO_data13.0']=merge([{'-s' : 'NANO:@PHYS+@MUPOG', '-n' : '1000'}, @@ -260,10 +269,13 @@ def subnext(self): workflows[_wfn()] = ['NANOmc130X', ['TTBarMINIAOD13.0', 'NANO_mc13.0', 'HRV_NANO_mc']] workflows[_wfn()] = ['EGMNANOmc130X', ['TTBarMINIAOD13.0', 'EGMNano_mc13.0']] _wfn.subnext() -workflows[_wfn()] = ['NANOdata130Xrun3', ['MuonEG2023MINIAOD13.0', 'NANO_data13.0', 'HRV_NANO_data']] -workflows[_wfn()] = ['NANOdata130Xrun3', ['MuonEG2023MINIAOD13.0', 'NANO_data13.0_prompt', 'HRV_NANO_data']] workflows[_wfn()] = ['muDPGNANO130Xrun3', ['ZMuSkim2023DRAWRECO13.0', 'muDPGNANO_data13.0']] workflows[_wfn()] = ['muDPGNANOBkg130Xrun3', ['ZeroBias2023DRAW13.0', 'muDPGNANOBkg_data13.0']] +workflows[_wfn()] = ['hcalDPGNANO130Xrun3', ['ZeroBias2023DRAW13.0', 'hcalDPGNANO_data13.0']] +workflows[_wfn()] = ['hcalDPGCalibNANO130Xrun3', ['TestEnablesEcalHcal2023C', 'hcalDPGCalibNANO_data13.0']] +_wfn.subnext() +workflows[_wfn()] = ['NANOdata130Xrun3', ['MuonEG2023MINIAOD13.0', 'NANO_data13.0', 'HRV_NANO_data']] +workflows[_wfn()] = ['NANOdata130Xrun3', ['MuonEG2023MINIAOD13.0', 'NANO_data13.0_prompt', 'HRV_NANO_data']] workflows[_wfn()] = ['muPOGNANO130Xrun3', ['MuonEG2023MINIAOD13.0', 'muPOGNANO_data13.0']] workflows[_wfn()] = ['EGMNANOdata130Xrun3', ['MuonEG2023MINIAOD13.0', 'EGMNano_data13.0']] workflows[_wfn()] = ['jmeNANOdata130Xrun3', ['MuonEG2023MINIAOD13.0', 'jmeNano_data13.0']] diff --git a/Configuration/PyReleaseValidation/python/relval_production.py b/Configuration/PyReleaseValidation/python/relval_production.py index 8b7ae7763725a..1c60228a2df8a 100644 --- a/Configuration/PyReleaseValidation/python/relval_production.py +++ b/Configuration/PyReleaseValidation/python/relval_production.py @@ -38,9 +38,6 @@ workflows[1052] = ['',['RunCosmics2022F','TIER0EXPCOSMICSRUN3','ALCAEXPCOSMICSRUN3','ALCAHARVDSIPIXELLAMCS']] workflows[1052.1] = ['',['RunSiPixelCalCosmics2022F','ALCASPLITSIPIXELLAMCS','ALCAHARVDSIPIXELLAMCS']] -workflows[1060.1] = ['',['RunZeroBias2022D', 'HcalNano']] -workflows[1060.2] = ['',['TestEnablesEcalHcal2023C', 'HcalNanoCalibGap']] - ## MC production test #workflows[1100] = [ '',[]] diff --git a/Configuration/PyReleaseValidation/python/relval_steps.py b/Configuration/PyReleaseValidation/python/relval_steps.py index 601ce6d3e1915..a9a5fae684a69 100644 --- a/Configuration/PyReleaseValidation/python/relval_steps.py +++ b/Configuration/PyReleaseValidation/python/relval_steps.py @@ -601,9 +601,6 @@ steps['RunMuonEG2023C']={'INPUT':InputInfo(dataSet='/MuonEG/Run2023C-v1/RAW',label='2023C',events=100000,location='STD', ls=Run2023C)} steps['RunParkingDoubleMuonLowMass2023C']={'INPUT':InputInfo(dataSet='/ParkingDoubleMuonLowMass0/Run2023C-v1/RAW',label='2023C',events=100000,location='STD', ls=Run2023C)} -Run2023C_LS500={368489: [[46,546]]} -steps['TestEnablesEcalHcal2023C']={'INPUT':InputInfo(dataSet='/TestEnablesEcalHcal/Run2023C-Express-v4/RAW',label='2023C',events=100000,location='STD', ls=Run2023C_LS500)} - Run2023D={369978: [[1, 800]]} steps['RunMuon2023D']={'INPUT':InputInfo(dataSet='/Muon0/Run2023D-v1/RAW',label='2023D',events=100000,location='STD', ls=Run2023D)} steps['RunZeroBias2023D']={'INPUT':InputInfo(dataSet='/ZeroBias/Run2023D-v1/RAW',label='2023D',events=100000,location='STD', ls=Run2023D)} @@ -4122,23 +4119,6 @@ def gen2023HiMix(fragment,howMuch): '--eventcontent':'NANOEDMAODSIM', '--filein':'file:step3_inMINIAODSIM.root'} -steps['HcalNano']={'-s':'RAW2DIGI,RECO,USER:DPGAnalysis/HcalNanoAOD/hcalNano_cff.hcalNanoTask', - '--conditions':'auto:run3_data_prompt', - '-n':'10', - '--era' : 'Run3', - '--geometry' : 'DB:Extended', - '--datatier':'NANOAOD', - '--eventcontent':'NANOAOD'} - -steps['HcalNanoCalibGap']={'-s':'RAW2DIGI,RECO,USER:DPGAnalysis/HcalNanoAOD/hcalNano_cff.hcalNanoTask', - '--conditions':'auto:run3_data_prompt', - '-n':'10', - '--era' : 'Run3', - '--geometry' : 'DB:Extended', - '--datatier':'NANOAOD', - '--eventcontent':'NANOAOD', - '--customise':'DPGAnalysis/HcalNanoAOD/hcalNano_cff.customiseHcalCalib', - '--processName': 'USER'} ################################################################################# ####From this line till the end of the file : From d704931dc0a2f36b01977310f9d3a43efdc20ad0 Mon Sep 17 00:00:00 2001 From: jean-roch Date: Wed, 20 Mar 2024 12:58:31 +0100 Subject: [PATCH 3/4] adapt to default cff in case of using @ mapping --- Configuration/Applications/python/ConfigBuilder.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Configuration/Applications/python/ConfigBuilder.py b/Configuration/Applications/python/ConfigBuilder.py index 8925ed90462e0..7b12ce09be6fb 100644 --- a/Configuration/Applications/python/ConfigBuilder.py +++ b/Configuration/Applications/python/ConfigBuilder.py @@ -1801,10 +1801,10 @@ def prepare_NANO(self, stepSpec = '' ): else: _nanoSeq = stepSpec - # create full specified sequence using autoNANO + # create full specified sequence using autoNANO from PhysicsTools.NanoAOD.autoNANO import autoNANO, expandNanoMapping # if not a autoNANO mapping, load an empty customization, which later will be converted into the default. - _nanoCustoms = _nanoSeq.split('+') if '@' in stepSpec else [''] + _nanoCustoms = _nanoSeq.split('+') if '@' in stepSpec else [''] _nanoSeq = _nanoSeq.split('+') expandNanoMapping(_nanoSeq, autoNANO, 'sequence') expandNanoMapping(_nanoCustoms, autoNANO, 'customize') @@ -1812,8 +1812,8 @@ def prepare_NANO(self, stepSpec = '' ): _nanoSeq = list(sorted(set(_nanoSeq), key=_nanoSeq.index)) _nanoCustoms = list(sorted(set(_nanoCustoms), key=_nanoCustoms.index)) # replace empty sequence with default - _nanoSeq = [seq if seq!='' else self.NANODefaultSeq for seq in _nanoSeq] - _nanoCustoms = [cust if cust!='' else self.NANODefaultCustom for cust in _nanoCustoms] + _nanoSeq = [seq if seq!='' else f"{self.NANODefaultCFF}.{self.NANODefaultSeq}" for seq in _nanoSeq] + _nanoCustoms = [cust if cust!='' else self.NANODefaultCustom for cust in _nanoCustoms] # build and inject the sequence if len(_nanoSeq) < 1 and '@' in stepSpec: raise Exception(f'The specified mapping: {stepSpec} generates an empty NANO sequence. Please provide a valid mapping') @@ -1828,10 +1828,10 @@ def prepare_NANO(self, stepSpec = '' ): print("NANO: scheduling:",_subSeq) _seqToSchedule.append(_subSeq) self.scheduleSequence('+'.join(_seqToSchedule), 'nanoAOD_step') - + # add the customisations for custom in _nanoCustoms: - custom_path = custom if '.' in custom else '.'.join([_nanoCff,custom]) + custom_path = custom if '.' in custom else '.'.join([self.NANODefaultCFF,custom]) # customization order can be important for NANO, here later specified customise take precedence self._options.customisation_file.append(custom_path) if self._options.hltProcess: From 214c409dbbe981016285058d2f96afaac72bde29 Mon Sep 17 00:00:00 2001 From: jean-roch Date: Thu, 21 Mar 2024 16:28:16 +0100 Subject: [PATCH 4/4] use specified cff when provided --- Configuration/Applications/python/ConfigBuilder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Configuration/Applications/python/ConfigBuilder.py b/Configuration/Applications/python/ConfigBuilder.py index 7b12ce09be6fb..4014e52efcf45 100644 --- a/Configuration/Applications/python/ConfigBuilder.py +++ b/Configuration/Applications/python/ConfigBuilder.py @@ -1800,7 +1800,9 @@ def prepare_NANO(self, stepSpec = '' ): _,_nanoSeq,_nanoCff = self.loadDefaultOrSpecifiedCFF(stepSpec,self.NANODefaultCFF,self.NANODefaultSeq) else: _nanoSeq = stepSpec + _nanoCff = self.NANODefaultCFF + print(_nanoSeq) # create full specified sequence using autoNANO from PhysicsTools.NanoAOD.autoNANO import autoNANO, expandNanoMapping # if not a autoNANO mapping, load an empty customization, which later will be converted into the default. @@ -1831,7 +1833,7 @@ def prepare_NANO(self, stepSpec = '' ): # add the customisations for custom in _nanoCustoms: - custom_path = custom if '.' in custom else '.'.join([self.NANODefaultCFF,custom]) + custom_path = custom if '.' in custom else '.'.join([_nanoCff,custom]) # customization order can be important for NANO, here later specified customise take precedence self._options.customisation_file.append(custom_path) if self._options.hltProcess: