Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Shorten upgrade workflow names, reorganize premixing workflow setup and add to PR tests for phase 2 #30988

Merged
merged 8 commits into from
Aug 4, 2020
Merged

Shorten upgrade workflow names, reorganize premixing workflow setup and add to PR tests for phase 2 #30988

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
c3ef67e
add phase 2 premixing workflow to PR tests
kpedro88 Jul 30, 2020
92849ea
reimplement premixing workflows using special workflow infrastructure…
kpedro88 Aug 1, 2020
674abf2
shorten names
kpedro88 Aug 1, 2020
1212aae
restrict new wf to phase2
kpedro88 Aug 1, 2020
4475c29
only display special workflow suffix once
kpedro88 Aug 3, 2020
53d36cd
avoid duplication of key in wf names
kpedro88 Aug 3, 2020
0ac121a
remove unnecessary parts of step names
kpedro88 Aug 3, 2020
aaf8a4a
faster search of keys dict
kpedro88 Aug 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions Configuration/PyReleaseValidation/python/MatrixInjector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,10 +264,10 @@ def prepare(self,mReader, directories, mode='init'):
wmsplit['RECOUP15']=1
wmsplit['RECOAODUP15']=5
wmsplit['DBLMINIAODMCUP15NODQM']=5
wmsplit['DigiFull']=5
wmsplit['RecoFull']=5
wmsplit['DigiFullPU']=1
wmsplit['RecoFullPU']=1
wmsplit['Digi']=5
wmsplit['Reco']=5
wmsplit['DigiPU']=1
wmsplit['RecoPU']=1
wmsplit['RECOHID11']=1
wmsplit['DIGIUP17']=1
wmsplit['RECOUP17']=1
Expand All @@ -289,8 +289,8 @@ def prepare(self,mReader, directories, mode='init'):
from .upgradeWorkflowComponents import upgradeKeys
for key in upgradeKeys[2026]:
if not "PU" in key: continue
wmsplit['DigiFullTriggerPU_'+key] = 1
wmsplit['RecoFullGlobalPU_'+key] = 1
wmsplit['DigiTriggerPU_'+key] = 1
wmsplit['RecoGlobalPU_'+key] = 1

#import pprint
#pprint.pprint(wmsplit)
Expand Down
20 changes: 17 additions & 3 deletions Configuration/PyReleaseValidation/python/MatrixReader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,13 @@ def readMatrix(self, fileNameIn, useInput=None, refRel=None, fromScratch=None):
wfName = wfInfo[0]
stepList = wfInfo[1]
stepOverrides=wfInfo.overrides
# upgrade case: workflow has basic name, key[, suffix (only special workflows)]
wfKey = ""
wfSuffix = ""
if isinstance(wfName, list) and len(wfName)>1:
if len(wfName)>2: wfSuffix = wfName[2]
wfKey = wfName[1]
wfName = wfName[0]
# if no explicit name given for the workflow, use the name of step1
if wfName.strip() == '': wfName = stepList[0]
# option to specialize the wf as the third item in the WF list
Expand All @@ -199,6 +206,10 @@ def readMatrix(self, fileNameIn, useInput=None, refRel=None, fromScratch=None):
addTo.append(0)

name=wfName
# separate suffixes by + because show() excludes first part of name
if len(wfKey)>0:
name = name+'+'+wfKey
if len(wfSuffix)>0: name = name+wfSuffix
stepIndex=0
ranStepList=[]

Expand All @@ -218,7 +229,7 @@ def readMatrix(self, fileNameIn, useInput=None, refRel=None, fromScratch=None):
else:
testName=step+'INPUT'
#print "JR",stepI,stepIr,testName,stepList
if testName in self.relvalModule.steps.keys():
if testName in self.relvalModule.steps:
#print "JR",stepI,stepIr
stepList[stepI]=testName
#pop the rest in the list
Expand Down Expand Up @@ -249,8 +260,11 @@ def readMatrix(self, fileNameIn, useInput=None, refRel=None, fromScratch=None):
stepName = step+"INPUT"
stepList.remove(step)
stepList.insert(stepIndex,stepName)
"""
name += stepName
"""
stepNameTmp = stepName
if len(wfKey)>0: stepNameTmp = stepNameTmp.replace('_'+wfKey,"")
if len(wfSuffix)>0: stepNameTmp = stepNameTmp.replace(wfSuffix,"")
name += stepNameTmp
if addCom and (not addTo or addTo[stepIndex]==1):
from Configuration.PyReleaseValidation.relval_steps import merge
copyStep=merge(addCom+[self.makeStep(self.relvalModule.steps[stepName],stepOverrides)])
Expand Down
2 changes: 1 addition & 1 deletion Configuration/PyReleaseValidation/python/relval_2026.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
numWFIB = []
numWFIB.extend([23234.0]) #2026D49
numWFIB.extend([23461.97]) #2026D49 premixing stage1 (NuGun+PU)
numWFIB.extend([23434.99]) #2026D49 premixing combined stage1+stage2 (ttbar+PU)
numWFIB.extend([23434.99,23434.999]) #2026D49 premixing combined stage1+stage2 (ttbar+PU200, ttbar+PU50 for PR test)
numWFIB.extend([23234.21,23434.21]) #2026D49 prodlike, prodlike PU
numWFIB.extend([23234.103]) #2026D49 aging
numWFIB.extend([23634.0]) #2026D51
Expand Down
107 changes: 22 additions & 85 deletions Configuration/PyReleaseValidation/python/relval_steps.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1752,16 +1752,8 @@ def lhegensim2018ml(fragment,howMuch):
'--procModifiers': 'premix_stage2',
'--era' : 'Run2_2016'
}
# Specifying explicitly the --filein is not nice but that was the
# easiest way to "skip" the output of step2 (=premixing stage1) for
# filein (as it goes to pileup_input). It works (a bit accidentally
# though) also for "-i all" because in that case the --filein for DAS
# input is after this one in the list of command line arguments to
# cmsDriver, and gets then used in practice.
digiPremixLocalPileup = {
"--filein": "file:step1.root",
"--pileup_input": "file:step2.root"
}

from .upgradeWorkflowComponents import digiPremixLocalPileup
digiPremixLocalPileupUp2015Defaults25ns = merge([digiPremixLocalPileup,
digiPremixUp2015Defaults25ns])
digiPremixUp2015Defaults50ns=merge([{'-s':'DIGI:pdigi_valid,DATAMIX,L1,DIGI2RAW,HLT:@relval50ns'},
Expand Down Expand Up @@ -3261,17 +3253,17 @@ def gen2021HiMix(fragment,howMuch):
beamspot=upgradeProperties[year][k].get('BeamSpot', None)

# setup baseline steps
upgradeStepDict['GenSimFull'][k]= {'-s' : 'GEN,SIM',
upgradeStepDict['GenSim'][k]= {'-s' : 'GEN,SIM',
'-n' : 10,
'--conditions' : gt,
'--beamspot' : 'Realistic25ns13TeVEarly2017Collision',
'--datatier' : 'GEN-SIM',
'--eventcontent': 'FEVTDEBUG',
'--geometry' : geom
}
if beamspot is not None: upgradeStepDict['GenSimFull'][k]['--beamspot']=beamspot
if beamspot is not None: upgradeStepDict['GenSim'][k]['--beamspot']=beamspot

upgradeStepDict['GenSimHLBeamSpotFull'][k]= {'-s' : 'GEN,SIM',
upgradeStepDict['GenSimHLBeamSpot'][k]= {'-s' : 'GEN,SIM',
'-n' : 10,
'--conditions' : gt,
'--beamspot' : 'HLLHC',
Expand All @@ -3280,7 +3272,7 @@ def gen2021HiMix(fragment,howMuch):
'--geometry' : geom
}

upgradeStepDict['GenSimHLBeamSpotFull14'][k]= {'-s' : 'GEN,SIM',
upgradeStepDict['GenSimHLBeamSpot14'][k]= {'-s' : 'GEN,SIM',
'-n' : 10,
'--conditions' : gt,
'--beamspot' : 'HLLHC14TeV',
Expand All @@ -3289,7 +3281,7 @@ def gen2021HiMix(fragment,howMuch):
'--geometry' : geom
}

upgradeStepDict['DigiFull'][k] = {'-s':'DIGI:pdigi_valid,L1,DIGI2RAW,HLT:%s'%(hltversion),
upgradeStepDict['Digi'][k] = {'-s':'DIGI:pdigi_valid,L1,DIGI2RAW,HLT:%s'%(hltversion),
'--conditions':gt,
'--datatier':'GEN-SIM-DIGI-RAW',
'-n':'10',
Expand All @@ -3298,57 +3290,57 @@ def gen2021HiMix(fragment,howMuch):
}

# Adding Track trigger step in step2
upgradeStepDict['DigiFullTrigger'][k] = {'-s':'DIGI:pdigi_valid,L1TrackTrigger,L1,DIGI2RAW,HLT:%s'%(hltversion),
upgradeStepDict['DigiTrigger'][k] = {'-s':'DIGI:pdigi_valid,L1TrackTrigger,L1,DIGI2RAW,HLT:%s'%(hltversion),
'--conditions':gt,
'--datatier':'GEN-SIM-DIGI-RAW',
'-n':'10',
'--eventcontent':'FEVTDEBUGHLT',
'--geometry' : geom
}

upgradeStepDict['RecoFull'][k] = {'-s':'RAW2DIGI,L1Reco,RECO,RECOSIM,EI,PAT,VALIDATION:@standardValidation+@miniAODValidation,DQM:@standardDQM+@ExtraHLT+@miniAODDQM',
upgradeStepDict['Reco'][k] = {'-s':'RAW2DIGI,L1Reco,RECO,RECOSIM,EI,PAT,VALIDATION:@standardValidation+@miniAODValidation,DQM:@standardDQM+@ExtraHLT+@miniAODDQM',
'--conditions':gt,
'--datatier':'GEN-SIM-RECO,MINIAODSIM,DQMIO',
'-n':'10',
'--eventcontent':'RECOSIM,MINIAODSIM,DQM',
'--geometry' : geom
}

upgradeStepDict['RecoFullGlobal'][k] = {'-s':'RAW2DIGI,L1Reco,RECO,RECOSIM,PAT,VALIDATION:@phase2Validation+@miniAODValidation,DQM:@phase2+@miniAODDQM',
upgradeStepDict['RecoGlobal'][k] = {'-s':'RAW2DIGI,L1Reco,RECO,RECOSIM,PAT,VALIDATION:@phase2Validation+@miniAODValidation,DQM:@phase2+@miniAODDQM',
'--conditions':gt,
'--datatier':'GEN-SIM-RECO,MINIAODSIM,DQMIO',
'-n':'10',
'--eventcontent':'FEVTDEBUGHLT,MINIAODSIM,DQM',
'--geometry' : geom
}

upgradeStepDict['RecoFullLocal'][k] = {'-s':'RAW2DIGI,L1Reco,RECO:localreco',
upgradeStepDict['RecoLocal'][k] = {'-s':'RAW2DIGI,L1Reco,RECO:localreco',
'--conditions':gt,
'--datatier':'GEN-SIM-RECO',
'-n':'10',
'--eventcontent':'FEVTDEBUGHLT',
'--geometry' : geom
}

upgradeStepDict['MiniAODFullGlobal'][k] = {'-s':'PAT',
upgradeStepDict['MiniAOD'][k] = {'-s':'PAT',
'--conditions':gt,
'--datatier':'MINIAODSIM',
'-n':'10',
'--eventcontent':'MINIAODSIM',
'--geometry' : geom
}

upgradeStepDict['HARVESTFull'][k]={'-s':'HARVESTING:@standardValidation+@standardDQM+@ExtraHLT+@miniAODValidation+@miniAODDQM',
upgradeStepDict['HARVEST'][k]={'-s':'HARVESTING:@standardValidation+@standardDQM+@ExtraHLT+@miniAODValidation+@miniAODDQM',
'--conditions':gt,
'--mc':'',
'--geometry' : geom,
'--scenario' : 'pp',
'--filetype':'DQM',
}

upgradeStepDict['HARVESTFullGlobal'][k] = merge([{'-s': 'HARVESTING:@phase2Validation+@phase2+@miniAODValidation+@miniAODDQM'}, upgradeStepDict['HARVESTFull'][k]])
upgradeStepDict['HARVESTGlobal'][k] = merge([{'-s': 'HARVESTING:@phase2Validation+@phase2+@miniAODValidation+@miniAODDQM'}, upgradeStepDict['HARVEST'][k]])

upgradeStepDict['ALCAFull'][k] = {'-s':'ALCA:SiPixelCalSingleMuon+TkAlMuonIsolated+TkAlMinBias+MuAlOverlaps+EcalESAlign+TkAlZMuMu+HcalCalHBHEMuonFilter+TkAlUpsilonMuMu+TkAlJpsiMuMu+SiStripCalMinBias',
upgradeStepDict['ALCA'][k] = {'-s':'ALCA:SiPixelCalSingleMuon+TkAlMuonIsolated+TkAlMinBias+MuAlOverlaps+EcalESAlign+TkAlZMuMu+HcalCalHBHEMuonFilter+TkAlUpsilonMuMu+TkAlJpsiMuMu+SiStripCalMinBias',
'--conditions':gt,
'--datatier':'ALCARECO',
'-n':'10',
Expand All @@ -3372,7 +3364,7 @@ def gen2021HiMix(fragment,howMuch):
'--scenario' : 'pp'
}

upgradeStepDict['NanoFull'][k] = {'-s':'NANO',
upgradeStepDict['Nano'][k] = {'-s':'NANO',
'--conditions':gt,
'--datatier':'NANOAODSIM',
'-n':'10',
Expand All @@ -3387,81 +3379,26 @@ def gen2021HiMix(fragment,howMuch):

# setup PU
if k2 in PUDataSets:
# Setup premixing stage1
#
# Has to be done before the overall PU definition in order to benefit from that
#
# It is a complete overkill to define premixing step for
# each generator fragment, but let's worry about
# simplification later
for step in upgradeWFs['baseline'].steps:
if "GenSim" in step:
stepNamePmx = step.replace('GenSim', 'Premix') + 'PU' + upgradeWFs['Premix'].suffix
d = merge([{'-s' : 'GEN,SIM,DIGI:pdigi_valid',
'--datatier' : 'PREMIX',
'--eventcontent': 'PREMIX',
'--procModifiers': 'premix_stage1',
},
PUDataSets[k2],upgradeStepDict[step][k]])
upgradeStepDict[stepNamePmx][k] = d

for specialType,specialWF in six.iteritems(upgradeWFs):
if "Premix" in specialType:
# Premix stage1 is already set above, and there are no non-PU steps so has to be ignored here
continue
for step in specialWF.PU:
stepName = step + specialWF.suffix
stepNamePU = step + 'PU' + specialWF.suffix
stepNamePUpmx = step + 'PUPRMX' + specialWF.suffix
stepName = specialWF.getStepName(step)
stepNamePU = specialWF.getStepNamePU(step)
if k not in upgradeStepDict[stepName] or upgradeStepDict[stepName][k] is None:
upgradeStepDict[stepNamePU][k] = None
elif stepNamePU in upgradeStepDict and k in upgradeStepDict[stepNamePU]:
# in case special WF had PU-specific changes
upgradeStepDict[stepNamePU][k]=merge([PUDataSets[k2],upgradeStepDict[stepNamePU][k]])
else:
upgradeStepDict[stepNamePU][k]=merge([PUDataSets[k2],upgradeStepDict[stepName][k]])

# Setup premixing stage2
if "Digi" in step or "Reco" in step:
d = merge([upgradeStepDict[stepName][k]])
if d is None: continue
if "Digi" in step:
tmpsteps = []
for s in d["-s"].split(","):
if s == "DIGI" or "DIGI:" in s:
tmpsteps.extend([s, "DATAMIX"])
else:
tmpsteps.append(s)
d = merge([{"-s" : ",".join(tmpsteps),
"--datamix" : "PreMix",
"--procModifiers": "premix_stage2"},
d])
elif "Reco" in step:
if "--procModifiers" in d:
d["--procModifiers"] += ",premix_stage2"
else:
d["--procModifiers"] = "premix_stage2"
upgradeStepDict[stepNamePUpmx][k] = d
# For combined stage1+stage2
if "Digi" in step:
upgradeStepDict[stepNamePUpmx+"Combined"][k] = merge([digiPremixLocalPileup, d])
# Increase the input file step number by one for Nano in combined stage1+stage2
if "Nano" in step:
d = merge([upgradeStepDict[stepName][k]])
if "--filein" in d:
filein = d["--filein"]
m = re.search("step(?P<ind>\d+)_", filein)
if m:
d["--filein"] = filein.replace(m.group(), "step%d_"%(int(m.group("ind"))+1))
upgradeStepDict[stepNamePUpmx+"Combined"][k] = d
# in case special WF has PU-specific changes: apply *after* basic PU step is created
specialWF.setupPU(upgradeStepDict, k, upgradeProperties[year][k])

for step in upgradeStepDict.keys():
# we need to do this for each fragment
if 'Sim' in step or 'Premix' in step:
for frag,info in six.iteritems(upgradeFragments):
howMuch=info.howMuch
for key in [key for year in upgradeKeys for key in upgradeKeys[year]]:
k=frag[:-4]+'_'+key+'_'+step
if (step in upgradeStepDict or step.replace("PUPRMX", "PU")) and key in upgradeStepDict[step]:
if step in upgradeStepDict and key in upgradeStepDict[step]:
if upgradeStepDict[step][key] is None:
steps[k]=None
elif 'Premix' in step:
Expand Down
Loading