From c75b6f6b1798cd2f602f931107b440175a3cfbb4 Mon Sep 17 00:00:00 2001
From: Brent Yates <yates.313@osu.edu>
Date: Wed, 3 Jul 2024 10:48:54 -0500
Subject: [PATCH 1/4] Updating to Python 3
---
mcgeneration/configure_gridpack.py | 41 +++++----
mcgeneration/helpers/CardEditor.py | 26 +++---
mcgeneration/helpers/Gridpack.py | 128 ++++++++++++++-------------
mcgeneration/helpers/JobTracker.py | 26 +++---
mcgeneration/helpers/ScanType.py | 2 +-
mcgeneration/helpers/helper_tools.py | 5 +-
6 files changed, 115 insertions(+), 113 deletions(-)
diff --git a/mcgeneration/configure_gridpack.py b/mcgeneration/configure_gridpack.py
index bb6d028..01f76e5 100644
--- a/mcgeneration/configure_gridpack.py
+++ b/mcgeneration/configure_gridpack.py
@@ -1,7 +1,6 @@
import os
import subprocess
import shutil
-import itertools
import random
import time
@@ -142,7 +141,7 @@
def cmsconnect_chain_submit(gridpack,dofs,proc_list,tag_postfix,rwgt_pts,runs,stype,scan_files=[],proc_run_wl={},attempt_resubmit=False):
#NOTE: The proc_run_wl is only use for SLINSPACE mode
if runs == 0:
- print "ERROR: For Batch jobs, need to specify at least 1 run!"
+ print("ERROR: For Batch jobs, need to specify at least 1 run!")
return
tracker = JobTracker(fdir=os.getcwd())
@@ -163,7 +162,7 @@ def cmsconnect_chain_submit(gridpack,dofs,proc_list,tag_postfix,rwgt_pts,runs,st
tracker.update()
tracker.showJobs(wl=[JobTracker.CODEGEN,JobTracker.INTEGRATE])
tracker.checkProgress()
- print ""
+ print("")
max_submits = min(
max_gen - len(tracker.codegen),
max_int - len(tracker.intg_filter),
@@ -263,16 +262,16 @@ def cmsconnect_chain_submit(gridpack,dofs,proc_list,tag_postfix,rwgt_pts,runs,st
)
if submitted >= max_submits:
break
- print ""
+ print("")
if not submitted and len(tracker.running) == 0:
# Nothing left to submit and all jobs have finished running
# Note: A kill command sent to the parent still wont kill the children processes
done = True
else:
time.sleep(delay)
- print "Done submitting jobs!"
- print "IMPORTANT: Make sure to check the condor_q for any held jobs!"
- #print "IMPORTANT: There could still be (soon to be orphaned) running jobs, make sure to check that they complete properly!"
+ print("Done submitting jobs!")
+ print("IMPORTANT: Make sure to check the condor_q for any held jobs!")
+ #print("IMPORTANT: There could still be (soon to be orphaned) running jobs, make sure to check that they complete properly!")
# Creates 1-D gridpacks at multiple linspaced starting points for each WC specified
def submit_1dim_jobs(gp,dofs,npts,runs,tag_postfix='',max_submits=-1,run_wl={}):
@@ -308,12 +307,12 @@ def submit_1dim_jobs(gp,dofs,npts,runs,tag_postfix='',max_submits=-1,run_wl={}):
)
if not gp.exists():
gp.setup()
- print gp.baseSettings(),
+ print(gp.baseSettings())
submitted += gp.submit()
time.sleep(delay)
- print ""
+ print("")
else:
- print "Skipping gridpack: %s" % (gp.getSetupString())
+ print("Skipping gridpack: %s" % (gp.getSetupString()))
if max_submits > 0 and submitted >= max_submits:
return submitted
return submitted
@@ -330,7 +329,7 @@ def submit_ndim_jobs(gp,dofs,npts,runs,tag,start_pts=[],max_submits=-1):
dof.setLimits(0,None,None)
pt = {}
if idx < len(start_pts):
- for k,v in start_pts[idx].iteritems(): pt[k] = v
+ for k,v in start_pts[idx].items(): pt[k] = v
gp.configure(
tag=tag,
run=idx,
@@ -340,12 +339,12 @@ def submit_ndim_jobs(gp,dofs,npts,runs,tag,start_pts=[],max_submits=-1):
)
if not gp.exists():
gp.setup()
- print gp.baseSettings(),
+ print(gp.baseSettings())
submitted += gp.submit()
time.sleep(delay)
- print ""
+ print("")
else:
- print "Skipping gridpack: %s" % (gp.getSetupString())
+ print("Skipping gridpack: %s" % (gp.getSetupString()))
if max_submits > 0 and submitted >= max_submits:
return submitted
return submitted
@@ -366,12 +365,12 @@ def submit_scanfile_jobs(gp,dofs,tag,scan_files,max_submits=-1):
)
if not gp.exists():
gp.setup()
- print gp.baseSettings(),
+ print(gp.baseSettings())
submitted += gp.submit()
time.sleep(delay)
- print ""
+ print("")
else:
- print "Skipping gridpack: %s" % (gp.getSetupString())
+ print("Skipping gridpack: %s" % (gp.getSetupString()))
if max_submits > 0 and submitted >= max_submits:
return submitted
return submitted
@@ -535,12 +534,12 @@ def main():
gridpack.configure(tag=tag,run=0,dofs=dof_list,num_pts=npts,start_pt=start_pt)
if not gridpack.exists():
gridpack.setup()
- print gridpack.baseSettings(),
+ print(gridpack.baseSettings())
submitted += gridpack.submit()
- print ""
+ print("")
else:
- print "Skipping gridpack: %s" % (gridpack.getSetupString())
+ print("Skipping gridpack: %s" % (gridpack.getSetupString()))
if __name__ == "__main__":
main()
- print "\nFinished!"
+ print("\nFinished!")
diff --git a/mcgeneration/helpers/CardEditor.py b/mcgeneration/helpers/CardEditor.py
index f22c0a7..18b5b9f 100644
--- a/mcgeneration/helpers/CardEditor.py
+++ b/mcgeneration/helpers/CardEditor.py
@@ -2,7 +2,7 @@
import shutil
import re
-from helper_tools import run_process
+from helpers.helper_tools import run_process
class BaseCard(object):
def __init__(self,card_dir,card_name):
@@ -47,12 +47,12 @@ def copy(self,dst,force=False):
s = "{src} and {dst} refer to the same file!".format(src=src,dst=dst)
raise RuntimeError(s)
if force:
- print "Removing existing file {dst}...".format(dst=dst)
+ print("Removing existing file {dst}...".format(dst=dst))
os.remove(dst)
else:
s = "{file} already exists!".format(file=dst)
raise RuntimeError(s)
- print "Copying to {dst}...".format(dst=dst)
+ print("Copying to {dst}...".format(dst=dst))
shutil.copyfile(src,dst)
def getFilePath(self):
@@ -66,7 +66,7 @@ def hasOption(self,k):
Checks if the option exists or not
k: The name of the option as it appears in the card
"""
- return self.__ops.has_key(k)
+ return k in self.__ops
def setOption(self,k,v):
"""
@@ -98,7 +98,7 @@ def dump(self):
"""
for k in self.list():
v = self.__ops[k]
- print "{0:>{w1}} = {1:<{w2}}".format(k,v,w1=self.key_width,w2=self.val_width)
+ print("{0:>{w1}} = {1:<{w2}}".format(k,v,w1=self.key_width,w2=self.val_width))
def parse(self):
"""
@@ -154,12 +154,12 @@ def dump(self):
w2=self.val_width,
w3=self.line_width
)
- print h
- print "-"*(len(h))
+ print(h)
+ print("-"*(len(h)))
for k in self.list():
v = self.getOption(k)
l = self.__line_map[k]
- print "{0:>{w1}} = {1:<{w2}} -- {2}".format(k,v,l,w1=self.key_width,w2=self.val_width)
+ print("{0:>{w1}} = {1:<{w2}} -- {2}".format(k,v,l,w1=self.key_width,w2=self.val_width))
def save(self,dst,force=False):
self.copy(dst,force=force)
@@ -179,7 +179,7 @@ def save(self,dst,force=False):
old = old.replace('\\','\\\\').replace('*','\\*')
new = new.replace('\\','\\\\').replace('*','\\*')
- #print "{old:<{w}} --> {new}".format(old=old,new=new,w=self.line_width)
+ #print("{old:<{w}} --> {new}".format(old=old,new=new,w=self.line_width))
sed_cmd = "s|{old}|{new}|g".format(old=old,new=new)
run_process(['sed','-i','-e',sed_cmd,dst])
@@ -209,7 +209,7 @@ def setOption(self,l):
def dump(self):
for k in self.list():
v = self.getOption(k)
- print "[{0:>{w1}}] {1:<{w2}}".format(k,v,w1=self.key_width,w2=self.val_width)
+ print("[{0:>{w1}}] {1:<{w2}}".format(k,v,w1=self.key_width,w2=self.val_width))
def save(self,dst,force=False,indent=0):
indent_str = " "*4*indent
@@ -218,12 +218,12 @@ def save(self,dst,force=False,indent=0):
s = "{ind}{dst} is not a file!".format(dst=dst,ind=indent_str)
raise RuntimeError(s)
if force:
- print "{ind}Removing {dst}...".format(dst=dst,ind=indent_str)
+ print("{ind}Removing {dst}...".format(dst=dst,ind=indent_str))
os.remove(dst)
else:
s = "{ind}{file} already exists!".format(file=dst,ind=indent_str)
raise RuntimeError(s)
- print "{ind}Saving to {dst}...".format(dst=dst,ind=indent_str)
+ print("{ind}Saving to {dst}...".format(dst=dst,ind=indent_str))
with open(dst,'w') as f:
for k in self.list():
v = self.getOption(k)
@@ -261,4 +261,4 @@ def test_customize_card():
if __name__ == "__main__":
test_run_card()
- test_customize_card()
\ No newline at end of file
+ test_customize_card()
diff --git a/mcgeneration/helpers/Gridpack.py b/mcgeneration/helpers/Gridpack.py
index d773de9..9931152 100644
--- a/mcgeneration/helpers/Gridpack.py
+++ b/mcgeneration/helpers/Gridpack.py
@@ -3,12 +3,12 @@
import shutil
import random
-from BatchType import BatchType
-from ScanType import ScanType
-from DegreeOfFreedom import DegreeOfFreedom
-from helper_tools import *
+from helpers.BatchType import BatchType
+from helpers.ScanType import ScanType
+from helpers.DegreeOfFreedom import DegreeOfFreedom
+from helpers.helper_tools import *
-from CardEditor import MGRunCard, MGCustomizeCard
+from helpers.CardEditor import MGRunCard, MGCustomizeCard
# Class for configuring and setting up the submission for a single gridpack, can also run a produced gridpack tarball
class Gridpack(object):
@@ -77,7 +77,7 @@ def __init__(self,**kwargs):
return
def hasOption(self,op_name):
- return self.ops.has_key(op_name)
+ return op_name in self.ops
# Note: Some options are complex objects (e.g. lists/dicts) and so the returned dict will contain
# only refs to those objects
@@ -94,9 +94,9 @@ def getOption(self,op):
return r
def setOptions(self,**kwargs):
- for op,v in kwargs.iteritems():
+ for op,v in kwargs.items():
if not self.hasOption(op):
- print "[ERROR] Unable to set option. Unknown Option: {op}".format(op=op)
+ print("[ERROR] Unable to set option. Unknown Option: {op}".format(op=op))
raise RuntimeError
self.ops[op] = v
@@ -119,9 +119,9 @@ def loadRunCard(self):
self.mg_runcard = MGRunCard(card_name=self.MG_RUN_CARD,card_dir=cdir)
def modifyRunCard(self,**ops):
- for op,val in ops.iteritems():
+ for op,val in ops.items():
if not self.mg_runcard.hasOption(op):
- print "[ERROR] Unknown run card option: {op}".format(op=op)
+ print("[ERROR] Unknown run card option: {op}".format(op=op))
raise RuntimeError
self.mg_runcard.setOption(op,val)
@@ -181,19 +181,19 @@ def saveProcessCard(self,indent=0):
if self.ops['save_diagrams']:
# Remove the nojpeg option from the output line of the process card
- print "{ind}Saving diagrams!".format(ind=indent_str)
+ print("{ind}Saving diagrams!".format(ind=indent_str))
run_process(['sed','-i','-e',"s|SUBSETUP -nojpeg|SUBSETUP|g",fpath])
if not self.ops['coupling_string'] is None:
# Replace the amp order specification with a new custom one
- print "{ind}Custom Couplings: {couplings}".format(couplings=self.ops['coupling_string'],ind=indent_str)
+ print("{ind}Custom Couplings: {couplings}".format(couplings=self.ops['coupling_string'],ind=indent_str))
sed_str = "s|DIM6=1|{new}|g".format(new=self.ops['coupling_string'])
run_process(['sed','-i','-e',sed_str,fpath])
if self.ops['use_coupling_model']:
# Replace the default dim6 model with the 'each_coupling_order' version
# NOTE: This will overwrite the 'replace_model' option
- print "{ind}Using each_coupling_order model!".format(ind=indent_str)
+ print("{ind}Using each_coupling_order model!".format(ind=indent_str))
old = "dim6top_LO_UFO"
new = "dim6top_LO_UFO_each_coupling_order"
sed_str = "s|import model {old}|import model {new}|g".format(old=old,new=new)
@@ -201,7 +201,7 @@ def saveProcessCard(self,indent=0):
if self.ops['replace_model']:
rep_model = self.ops['replace_model']
- print "{ind}Using {model} model".format(model=rep_model,ind=indent_str)
+ print("{ind}Using {model} model".format(model=rep_model,ind=indent_str))
old = "dim6top_LO_UFO"
sed_str = "s|import model {old}|import model {new}|g".format(old=old,new=rep_model)
run_process(['sed','-i','-e',sed_str,fpath])
@@ -298,7 +298,7 @@ def limitSettings(self,header=True,depth=0):
indent = "\t"*depth
info = ""
if header: info += indent + "Limit Settings: %s\n" % (self.getSetupString())
- for c,dof in self.ops['coeffs'].iteritems():
+ for c,dof in self.ops['coeffs'].items():
key = "%s_%s" % (self.ops['limits_name'],c)
if header: info += "\t"
info += indent + "%s: [" % (key.ljust(11))
@@ -400,7 +400,7 @@ def exists(self):
def configure(self,tag,run,dofs,num_pts,start_pt={},scan_file=None):
""" Parses options to produce gridpack in a particular way. """
if len(self.ops['default_limits']) != 2:
- print "Invalid input for default limits!"
+ print("Invalid input for default limits!")
self.is_configured = False
return
@@ -419,16 +419,17 @@ def configure(self,tag,run,dofs,num_pts,start_pt={},scan_file=None):
extra_wc = []
for idx,pt in enumerate(pts):
new_pt = {}
- for k,v in pt.iteritems():
+ for k,v in pt.items():
# Keep only the WCs which have been specified
- if self.ops['coeffs'].has_key(k):
+ if k in self.ops['coeffs']:
new_pt[k] = v
else:
extra_wc.append(k)
if idx == 0:
# Set the starting point from the scanpoints file
for k in self.ops['coeffs'].keys():
- if not new_pt.has_key(k):
+ if not k in new_pt:
+ #if not new_pt.has_key(k):
# Any WCs which are missing from the scanpoints file are set to SM value
missing_wc.append(k)
self.ops['coeffs'][k].setLimits(0,0,0)
@@ -437,10 +438,10 @@ def configure(self,tag,run,dofs,num_pts,start_pt={},scan_file=None):
else:
self.scan_pts.append(new_pt)
if len(missing_wc):
- print "[WARNING] Scanpoints file is missing WCs used in this gridpack configuration, %s" % (str(missing_wc))
+ print("[WARNING] Scanpoints file is missing WCs used in this gridpack configuration, %s" % (str(missing_wc)))
if len(extra_wc):
- print "[WARNING] Scanpoints file has WCs that were not specified in this gridpack configuration, their values will be set to SM."
- print "\t%s" % (str(extra_wc))
+ print("[WARNING] Scanpoints file has WCs that were not specified in this gridpack configuration, their values will be set to SM.")
+ print("\t%s" % (str(extra_wc)) )
self.ops['num_rwgt_pts'] = len(self.scan_pts)
else:
self.scan_pts = [] # Clear the scan_pts array incase it was used previously
@@ -464,14 +465,16 @@ def configure(self,tag,run,dofs,num_pts,start_pt={},scan_file=None):
# The dof already has limits set
continue
key = "%s_%s" % (self.ops['limits_name'],c)
- if wc_limits.has_key(key):
+ if key in wc_limits:
+ #if wc_limits.has_key(key):
# Use limits based on those found in the limits file
low = round(wc_limits[key][0],6)
high = round(wc_limits[key][1],6)
else:
# The WC doesn't exist in the limits file, so use defaults
low,high = self.ops['default_limits']
- if start_pt.has_key(c):
+ if c in start_pt:
+ #if start_pt.has_key(c):
strength = start_pt[c]
else:
strength = calculate_start_point(low,high,1.25)
@@ -491,29 +494,29 @@ def setup(self,indent=0):
setup = self.getSetupString()
if self.exists():
- print "{0:>{w}}Skipping gridpack setup: {setup}".format("",setup=setup,w=4*indent)
+ print("{0:>{w}}Skipping gridpack setup: {setup}".format("",setup=setup,w=4*indent))
return False
if self.ops['save_diagrams'] and self.ops['btype'] != BatchType.LOCAL:
- print "[ERROR] Invalid BatchType for saving diagrams: {btype}".format(btype=self.ops['btype'])
+ print("[ERROR] Invalid BatchType for saving diagrams: {btype}".format(btype=self.ops['btype']))
return False
if self.ops['stype'] == ScanType.NONE:
# Don't do any reweighting
self.scan_pts = []
- print "{0:>{w}}Setup gridpack: {setup}...".format("",setup=setup,w=4*indent)
+ print("{0:>{w}}Setup gridpack: {setup}...".format("",setup=setup,w=4*indent))
# Set the random seed adding extra events to the pilotrun
# NOTE: This physically modifies the gridpack generation script
seed = int(random.uniform(1,1e6))
- print "{ind:>{w}}Seed: {seed:d}".format(seed=seed,ind="",w=4*(indent+1))
+ print("{ind:>{w}}Seed: {seed:d}".format(seed=seed,ind="",w=4*(indent+1)))
sed_str = "s|RWSEED=[0-9]*|RWSEED={seed:d}|g".format(seed=seed)
run_process(['sed','-i','-e',sed_str,self.GENPROD_SCRIPT])
target_dir = self.getTargetDirectory(create=False)
if os.path.exists(target_dir):
- print "{0:>{w}}NOTE: The cards directory already exists, will overwrite existing cards.".format("",w=4*(indent+1))
+ print("{0:>{w}}NOTE: The cards directory already exists, will overwrite existing cards.".format("",w=4*(indent+1)))
else:
# Create the needed directories
self.getTargetDirectory(create=True)
@@ -537,8 +540,8 @@ def setup(self,indent=0):
if self.ops['flavor_scheme'] == 5:
extra_customize_ops.append('set param_card MB 0.0')
extra_customize_ops.append('set param_card ymb 0.0')
- for c,dof in self.ops['coeffs'].iteritems():
- for k,v in dof.eval(dof.getStart()).iteritems():
+ for c,dof in self.ops['coeffs'].items():
+ for k,v in dof.eval(dof.getStart()).items():
extra_customize_ops.append('set param_card {wc} {val:.6f}'.format(wc=k,val=v))
self.modifyCustomizeCard(*extra_customize_ops)
self.modifyRunCard(**self.ops['runcard_ops'])
@@ -560,17 +563,17 @@ def setup(self,indent=0):
#if self.ops['save_diagrams']:
# # Remove the nojpeg option from the output line of the process card
- # print "\tSaving diagrams!"
+ # print("\tSaving diagrams!")
# run_process(['sed','-i','-e',"s|SUBSETUP -nojpeg|SUBSETUP|g",proc_tar])
#if not self.ops['coupling_string'] is None:
- # print "\tCustom Couplings: %s" % (self.ops['coupling_string'])
+ # print("\tCustom Couplings: %s" % (self.ops['coupling_string']))
# run_process(['sed','-i','-e',"s|DIM6=1|%s|g" % (self.ops['coupling_string']),proc_tar])
#if self.ops['use_coupling_model']:
- # print "\tUsing each_coupling_order model!"
+ # print("\tUsing each_coupling_order model!")
# run_process(['sed','-i','-e',"s|import model dim6top_LO_UFO|import model dim6top_LO_UFO_each_coupling_order|g",proc_tar])
#if self.ops['replace_model']:
# rep_model = self.ops['replace_model']
- # print "\tUsing %s model" % (rep_model)
+ # print("\tUsing %s model" % (rep_model))
# run_process(['sed','-i','-e',"s|import model dim6top_LO_UFO|import model %s|g" % (rep_model),proc_tar])
## Replace SUBSETUP in the process card
#run_process(['sed','-i','-e',"s|SUBSETUP|%s|g" % (setup),proc_tar])
@@ -579,58 +582,58 @@ def setup(self,indent=0):
def clean(self):
""" Remove all folders/files created by the setup()/submit() methods """
if not self.is_configured:
- print "The gridpack has not been configured yet, so no cleaning can be done!"
+ print("The gridpack has not been configured yet, so no cleaning can be done!")
return
os.chdir(self.HOME_DIR)
- print "Cleaning files related to current gridpack configuration: %s" % (self.getSetupString())
+ print("Cleaning files related to current gridpack configuration: %s" % (self.getSetupString()))
target_dir = self.getTargetDirectory(create=False)
if os.path.exists(target_dir) and os.path.isdir(target_dir):
# This is where the modified madgraph cards are stored
- print "\tRemoving existing directory: %s " % (target_dir)
+ print("\tRemoving existing directory: %s " % (target_dir))
shutil.rmtree(target_dir)
setup_dir = self.getSetupString()
if os.path.exists(setup_dir) and os.path.isdir(setup_dir):
# This is the directory that gets created by the setup_production.sh script (in LOCAL or CMSCONNECT mode)
- print "\tRemoving existing directory: %s " % (setup_dir)
+ print("\tRemoving existing directory: %s " % (setup_dir))
shutil.rmtree(setup_dir)
gridrun_dir = self.getGridrunOutputDirectory()
if os.path.exists(gridrun_dir) and os.path.isdir(gridrun_dir):
# This is the directory that is used to unpack and run a gridpack tarball
- print "\tRemoving existing directory: %s " % (gridrun_dir)
+ print("\tRemoving existing directory: %s " % (gridrun_dir))
shutil.rmtree(gridrun_dir)
tarball_file = self.getTarballString()
if os.path.exists(tarball_file) and not os.path.isdir(tarball_file):
# This is the tarball created by the setup_production.sh script
- print "\tRemoving existing file: %s" % (tarball_file)
+ print("\tRemoving existing file: %s" % (tarball_file))
os.remove(tarball_file)
scanpoints_file = self.getScanfileString()
if os.path.exists(scanpoints_file) and not os.path.isdir(scanpoints_file):
# This is the txt file which contains the recorded starting point and list of madgraph rwgt points
- print "\tRemoving existing file: %s" % (scanpoints_file)
+ print("\tRemoving existing file: %s" % (scanpoints_file))
os.remove(scanpoints_file)
log_file = "%s.log" % (self.getSetupString())
if os.path.exists(log_file) and not os.path.isdir(log_file):
# This is the log file created by the setup_production.sh script
- print "\tRemoving existing file: %s" % (log_file)
+ print("\tRemoving existing file: %s" % (log_file))
os.remove(log_file)
debug_file = "%s.debug" % (self.getSetupString())
if os.path.exists(debug_file) and not os.path.isdir(debug_file):
# This is the debug file created by the setup_production.sh script (only in CMSCONNECT mode)
- print "\tRemoving existing file: %s" % (debug_file)
+ print("\tRemoving existing file: %s" % (debug_file))
os.remove(debug_file)
codegen_file = "%s_codegen.log" % (self.getSetupString())
if os.path.exists(codegen_file) and not os.path.isdir(codegen_file):
# This is the codegen log file created by the setup_production.sh script (only in CMSCONNECT mode)
- print "\tRemoving existing file: %s" % (codegen_file)
+ print("\tRemoving existing file: %s" % (codegen_file))
os.remove(codegen_file)
self.is_configured = False
@@ -641,10 +644,10 @@ def submit(self):
target_dir = self.getTargetDirectory()
btype = self.ops['btype']
if not os.path.exists(target_dir):
- print "[ERROR] Can't find target directory, %s" % (target_dir)
+ print("[ERROR] Can't find target directory, %s" % (target_dir))
return False
- print "Submit gridpack: %s..." % (setup)
- print "\tBatchType: %s" % (btype)
+ print("Submit gridpack: %s..." % (setup))
+ print("\tBatchType: %s" % (btype))
if btype == BatchType.LOCAL:
# For interactive/serial running
if self.ops['save_diagrams']:
@@ -660,15 +663,16 @@ def submit(self):
# For cmsconnect running
debug_file = "%s.debug" % (setup)
cmsconnect_cores = 1
- print '\tCurrent PATH: {0}'.format(os.getcwd())
- print '\tWill execute: ./submit_cmsconnect_gridpack_generation.sh {setup} {dir} {cores} "{mem}" {arch} {release}'.format(
+ print('\tCurrent PATH: {0}'.format(os.getcwd()))
+ print('\tWill execute: ./submit_cmsconnect_gridpack_generation.sh {setup} {dir} {cores} "{mem}" {arch} {release} {disk}'.format(
setup=setup,
dir=target_dir,
cores=str(cmsconnect_cores),
mem="15 Gb",
+ disk="15 Gb",
arch=self.CURR_ARCH,
release=self.CURR_RELEASE
- )
+ ))
subprocess.Popen(
["./submit_cmsconnect_gridpack_generation.sh",setup,target_dir,str(cmsconnect_cores),"15 Gb",self.CURR_ARCH,self.CURR_RELEASE],
stdout=open(debug_file,'w'),
@@ -677,11 +681,11 @@ def submit(self):
return True
elif btype == BatchType.CONDOR:
# Not currently working
- print "\tCondor running is not currently working. Sorry!"
+ print("\tCondor running is not currently working. Sorry!")
#run_process(['./submit_condor_gridpack_generation.sh',setup,target_dir])
return True
elif btype == BatchType.NONE:
- print "\tSkipping gridpack generation, %s" % (setup)
+ print("\tSkipping gridpack generation, %s" % (setup))
return True
return False
@@ -690,37 +694,37 @@ def run(self,events,seed,cores):
os.chdir(self.HOME_DIR)
setup = self.getSetupString()
- print "Running Gridpack: %s" % (setup)
- print "\tSetting up directories..."
+ print("Running Gridpack: %s" % (setup))
+ print("\tSetting up directories...")
output_dir = self.getGridrunOutputDirectory(create=False)
if os.path.exists(output_dir):
- print "Removing existing output directory: %s" % (output_dir)
+ print("Removing existing output directory: %s" % (output_dir))
shutil.rmtree(output_dir)
output_dir = self.getGridrunOutputDirectory(create=True)
#if os.path.exists(output_dir):
# # We already ran the gridpack once!
- # print "Output directory already exists, skipping gridpack run: %s" % (setup)
+ # print("Output directory already exists, skipping gridpack run: %s" % (setup))
# return
#else:
# os.mkdir(output_dir)
tarball = self.getTarballString()
if not os.path.exists(tarball):
- print "No tarball file found! Skipping..."
+ print("No tarball file found! Skipping...")
return
- #print "\tMoving tarball..."
+ #print("\tMoving tarball...")
#shutil.move(tarball,output_dir)
- print "\tExtracting tarball..."
+ print("\tExtracting tarball...")
run_process(['tar','xaf',tarball,'-C',output_dir])
os.chdir(output_dir)
- #print "\tExtracting tarball..."
+ #print("\tExtracting tarball...")
#run_process(['tar','xaf',tarball])
- print "\tRunning gridpack..."
+ print("\tRunning gridpack...")
run_process(['./runcmsgrid.sh',str(events),str(seed),str(cores)])
os.chdir(self.HOME_DIR)
diff --git a/mcgeneration/helpers/JobTracker.py b/mcgeneration/helpers/JobTracker.py
index dc8f126..87c5d40 100644
--- a/mcgeneration/helpers/JobTracker.py
+++ b/mcgeneration/helpers/JobTracker.py
@@ -1,7 +1,7 @@
import os
import datetime
import math
-from helper_tools import run_process,regex_match
+from helpers.helper_tools import run_process,regex_match
# Utility class for keeping track of gridpack production jobs
# NOTE: This assumes that all the relevant log files are in the same directory
@@ -58,7 +58,7 @@ def update(self):
self.stuck = self.getStuckJobs(self.stuck_cutoff)
def addResubmit(self,job):
- if not self.resubmitted.has_key(job):
+ if not job in self.resubmitted:
self.resubmitted[job] = 0
self.resubmitted[job] += 1
@@ -326,25 +326,25 @@ def checkProgress(self,lines=5):
t = self.getLastModifiedTime(log_file)
h,m,s = self.formatTime(t)
mod_tstr = "[%s:%s:%s]" % (h.rjust(2,"0"),m.rjust(2,"0"),s.rjust(2,"0"))
- #print "\nChecking: %s - %s - %s" % (fn,int_tstr,mod_tstr)
- #print "\nChecking: %s - Total %s - Intg %s - LogMod %s" % (fn,tot_tstr,int_tstr,mod_tstr)
- print "\nChecking: %s - %s - %s - %s" % (fn,tot_tstr,int_tstr,mod_tstr)
+ #print("\nChecking: %s - %s - %s" % (fn,int_tstr,mod_tstr))
+ #print("\nChecking: %s - Total %s - Intg %s - LogMod %s" % (fn,tot_tstr,int_tstr,mod_tstr))
+ print("\nChecking: %s - %s - %s - %s" % (fn,tot_tstr,int_tstr,mod_tstr))
run_process(['tail','-n%d' % (lines),log_file])
def displayJobList(self,s,arr):
- print "%s Jobs: %d" % (s,len(arr))
+ print("%s Jobs: %d" % (s,len(arr)))
for f in sorted(arr):
- print "\t%s" % (f)
+ print("\t%s" % (f))
def showJobs(self,wl=[]):
- print "Last Update: %s" % (self.last_update)
+ print("Last Update: %s" % (self.last_update))
if len(wl) == 0:
wl = self.getJobTypes()
if len(self.resubmitted):
- print "Resubmitted Jobs:"
+ print("Resubmitted Jobs:")
for k,v in self.resubmitted.iteritems():
- print "\t%s: %d" % (k,v)
+ print("\t%s: %d" % (k,v))
if self.RUNNING in wl:
self.displayJobList("Running", self.running)
@@ -372,9 +372,9 @@ def showJobs(self,wl=[]):
job_list = [JobTracker.RUNNING,JobTracker.CODEGEN,JobTracker.INTEGRATE,JobTracker.INTEGRATE_FILTER,JobTracker.STUCK]
tracker.showJobs(wl=job_list)
tracker.checkProgress(lines=5)
- print "\nChecking finished jobs for failures:"
+ print("\nChecking finished jobs for failures:")
for job in tracker.finished:
if tracker.logHasError(job=job,fdir=curr_dir):
- print "\tFinished Job %s has error!" % (job)
+ print("\tFinished Job %s has error!" % (job))
if not tracker.logHasXsec(job=job,fdir=curr_dir):
- print "\tFinished Job %s is missing xsec!" % (job)
+ print("\tFinished Job %s is missing xsec!" % (job))
diff --git a/mcgeneration/helpers/ScanType.py b/mcgeneration/helpers/ScanType.py
index cbe1052..4b510f3 100644
--- a/mcgeneration/helpers/ScanType.py
+++ b/mcgeneration/helpers/ScanType.py
@@ -1,6 +1,6 @@
import random
-from helper_tools import linspace, check_point
+from helpers.helper_tools import linspace, check_point
class ScanType(object):
FRANDOM = 'full_random'
diff --git a/mcgeneration/helpers/helper_tools.py b/mcgeneration/helpers/helper_tools.py
index 5a7fb69..bdb378b 100644
--- a/mcgeneration/helpers/helper_tools.py
+++ b/mcgeneration/helpers/helper_tools.py
@@ -1,4 +1,3 @@
-import itertools
import random
import subprocess
import shutil
@@ -16,7 +15,7 @@ def run_process(inputs,verbose=True,indent=0):
break
if l:
stdout.append(l.strip())
- if verbose: print indent_str+l.strip()
+ if verbose: print(indent_str+l.strip())
return stdout
def find_process(p_name,p_lst):
@@ -185,4 +184,4 @@ def regex_match(lst,regex_lst):
return matches
if __name__ == "__main__":
- pass
\ No newline at end of file
+ pass
From 7315642ac68232edfeb32989e44e4c547823f5e9 Mon Sep 17 00:00:00 2001
From: Andrew Wightman <awightma@login.uscms.org>
Date: Wed, 10 Jul 2024 18:10:30 -0500
Subject: [PATCH 2/4] Additional py2 to py3 fixes
---
mcgeneration/helpers/DegreeOfFreedom.py | 6 +++---
mcgeneration/helpers/Gridpack.py | 16 +++++++--------
mcgeneration/helpers/JobTracker.py | 2 +-
mcgeneration/helpers/ScanType.py | 8 ++++----
mcgeneration/helpers/helper_tools.py | 26 ++++++++++++-------------
5 files changed, 29 insertions(+), 29 deletions(-)
diff --git a/mcgeneration/helpers/DegreeOfFreedom.py b/mcgeneration/helpers/DegreeOfFreedom.py
index 28d098c..b12e0bf 100644
--- a/mcgeneration/helpers/DegreeOfFreedom.py
+++ b/mcgeneration/helpers/DegreeOfFreedom.py
@@ -11,7 +11,7 @@ def getName(self):
return self.name
def getCoefficients(self):
- return self.relations.keys()
+ return list(self.relations.keys())
def getStart(self):
return self.limits[0]
@@ -37,7 +37,7 @@ def popCoefficient(self,wc_name):
def eval(self,x):
output = {}
- for wc,scale in self.relations.iteritems():
+ for wc,scale in self.relations.items():
if x*scale == 0.0:
output[wc] = 0.0
else:
@@ -46,7 +46,7 @@ def eval(self,x):
def getCouplingString(self):
s = "FCNC=0 DIM6^2==1"
- for k in self.relations.keys():
+ for k in list(self.relations.keys()):
s += " DIM6_%s^2==1" % (k)
return s
diff --git a/mcgeneration/helpers/Gridpack.py b/mcgeneration/helpers/Gridpack.py
index 9931152..bce3719 100644
--- a/mcgeneration/helpers/Gridpack.py
+++ b/mcgeneration/helpers/Gridpack.py
@@ -94,7 +94,7 @@ def getOption(self,op):
return r
def setOptions(self,**kwargs):
- for op,v in kwargs.items():
+ for op,v in list(kwargs.items()):
if not self.hasOption(op):
print("[ERROR] Unable to set option. Unknown Option: {op}".format(op=op))
raise RuntimeError
@@ -119,7 +119,7 @@ def loadRunCard(self):
self.mg_runcard = MGRunCard(card_name=self.MG_RUN_CARD,card_dir=cdir)
def modifyRunCard(self,**ops):
- for op,val in ops.items():
+ for op,val in list(ops.items()):
if not self.mg_runcard.hasOption(op):
print("[ERROR] Unknown run card option: {op}".format(op=op))
raise RuntimeError
@@ -298,7 +298,7 @@ def limitSettings(self,header=True,depth=0):
indent = "\t"*depth
info = ""
if header: info += indent + "Limit Settings: %s\n" % (self.getSetupString())
- for c,dof in self.ops['coeffs'].items():
+ for c,dof in list(self.ops['coeffs'].items()):
key = "%s_%s" % (self.ops['limits_name'],c)
if header: info += "\t"
info += indent + "%s: [" % (key.ljust(11))
@@ -419,7 +419,7 @@ def configure(self,tag,run,dofs,num_pts,start_pt={},scan_file=None):
extra_wc = []
for idx,pt in enumerate(pts):
new_pt = {}
- for k,v in pt.items():
+ for k,v in list(pt.items()):
# Keep only the WCs which have been specified
if k in self.ops['coeffs']:
new_pt[k] = v
@@ -427,7 +427,7 @@ def configure(self,tag,run,dofs,num_pts,start_pt={},scan_file=None):
extra_wc.append(k)
if idx == 0:
# Set the starting point from the scanpoints file
- for k in self.ops['coeffs'].keys():
+ for k in list(self.ops['coeffs'].keys()):
if not k in new_pt:
#if not new_pt.has_key(k):
# Any WCs which are missing from the scanpoints file are set to SM value
@@ -451,7 +451,7 @@ def configure(self,tag,run,dofs,num_pts,start_pt={},scan_file=None):
if num_pts > 0:
# Make sure we have enough points to reconstruct the parametrization
if self.ops['stype'] == ScanType.FRANDOM:
- N = len(self.ops['coeffs'].keys())
+ N = len(list(self.ops['coeffs'].keys()))
num_pts = max(num_pts,1.2*(1+2*N+N*(N-1)/2))
elif self.ops['stype'] == ScanType.SLINSPACE:
num_pts = max(num_pts,3)
@@ -540,8 +540,8 @@ def setup(self,indent=0):
if self.ops['flavor_scheme'] == 5:
extra_customize_ops.append('set param_card MB 0.0')
extra_customize_ops.append('set param_card ymb 0.0')
- for c,dof in self.ops['coeffs'].items():
- for k,v in dof.eval(dof.getStart()).items():
+ for c,dof in list(self.ops['coeffs'].items()):
+ for k,v in list(dof.eval(dof.getStart()).items()):
extra_customize_ops.append('set param_card {wc} {val:.6f}'.format(wc=k,val=v))
self.modifyCustomizeCard(*extra_customize_ops)
self.modifyRunCard(**self.ops['runcard_ops'])
diff --git a/mcgeneration/helpers/JobTracker.py b/mcgeneration/helpers/JobTracker.py
index 87c5d40..b97c281 100644
--- a/mcgeneration/helpers/JobTracker.py
+++ b/mcgeneration/helpers/JobTracker.py
@@ -343,7 +343,7 @@ def showJobs(self,wl=[]):
if len(self.resubmitted):
print("Resubmitted Jobs:")
- for k,v in self.resubmitted.iteritems():
+ for k,v in self.resubmitted.items():
print("\t%s: %d" % (k,v))
if self.RUNNING in wl:
diff --git a/mcgeneration/helpers/ScanType.py b/mcgeneration/helpers/ScanType.py
index 4b510f3..60cb1b5 100644
--- a/mcgeneration/helpers/ScanType.py
+++ b/mcgeneration/helpers/ScanType.py
@@ -49,7 +49,7 @@ def fullScanLinear(cls,dofs,npts):
return pts_arr
sm_pt = {}
start_pt = {}
- coeffs = dofs.keys()
+ coeffs = list(dofs.keys())
arr = []
for c in coeffs:
sm_pt[c] = 0.0
@@ -83,7 +83,7 @@ def fullScanRandom(cls,dofs,npts):
return pts_arr
sm_pt = {}
start_pt = {}
- coeffs = dofs.keys()
+ coeffs = list(dofs.keys())
for c in coeffs:
sm_pt[c] = 0.0
start_pt[c] = dofs[c].getStart()
@@ -112,7 +112,7 @@ def axisScanLinear(cls,dofs,npts):
return pts_arr
sm_pt = {}
start_pt = {}
- coeffs = dofs.keys()
+ coeffs = list(dofs.keys())
for c in coeffs:
sm_pt[c] = 0.0
start_pt[c] = dofs[c].getStart()
@@ -146,7 +146,7 @@ def axisScanRandom(cls,dofs,npts):
return pts_arr
sm_pt = {}
start_pt = {}
- coeffs = dofs.keys()
+ coeffs = list(dofs.keys())
for c in coeffs:
sm_pt[c] = 0.0
start_pt[c] = dofs[c].getStart()
diff --git a/mcgeneration/helpers/helper_tools.py b/mcgeneration/helpers/helper_tools.py
index bdb378b..c72edff 100644
--- a/mcgeneration/helpers/helper_tools.py
+++ b/mcgeneration/helpers/helper_tools.py
@@ -45,8 +45,8 @@ def linspace(start,stop,num,endpoint=True,acc=7):
# Checks if two W.C. phase space points are identical
def check_point(pt1,pt2):
- for k,v in pt1.iteritems():
- if not pt2.has_key(k):
+ for k,v in pt1.items():
+ if k not in pt2:
pt2[k] = 0.0 # pt2 is missing the coeff, add it and set it to SM value
if v != pt2[k]:
return False
@@ -58,8 +58,8 @@ def set_initial_point(file_name,dofs,flavor_scheme=5):
if flavor_scheme == 5:
f.write('set param_card MB 0.0 \n')
f.write('set param_card ymb 0.0 \n')
- for c,dof in dofs.iteritems():
- for k,v in dof.eval(dof.getStart()).iteritems():
+ for c,dof in dofs.items():
+ for k,v in dof.eval(dof.getStart()).items():
f.write("set param_card %s %.6f \n" % (k,v))
f.write('\n')
return file_name
@@ -80,18 +80,18 @@ def make_reweight_card(file_name,dofs,pts):
f.write(header)
# This is a workaround for the MG bug causing first point to not be renamed
f.write("\nlaunch --rwgt_name=dummy_point")
- c = dofs.keys()[0]
- for k,v in dofs[c].eval(0.0123).iteritems():
+ c = list(dofs.keys())[0]
+ for k,v in dofs[c].eval(0.0123).items():
f.write("\nset %s %.6f" % (k,v))
f.write("\n")
for idx,pt in enumerate(pts):
rwgt_str = "EFTrwgt%d" % (idx)
- for k,v in pt.iteritems():
+ for k,v in pt.items():
rwgt_str += '_' + k + '_' + str(round(v,6))
f.write("\nlaunch --rwgt_name=%s" % (rwgt_str))
- for k1,v1 in pt.iteritems():
- for k2,v2 in dofs[k1].eval(v1).iteritems():
+ for k1,v1 in pt.items():
+ for k2,v2 in dofs[k1].eval(v1).items():
f.write("\nset %s %.6f" % (k2,v2))
f.write("\n")
@@ -151,18 +151,18 @@ def save_scan_points(fpath,dofs,rwgt_pts):
col_sep = " "
with open(fpath,'w') as f:
header = "".ljust(col_spacing)
- for k,dof in dofs.iteritems():
+ for k,dof in dofs.items():
header += dof.getName().ljust(col_spacing) + col_sep
start_row = "\nMGStart".ljust(col_spacing) + col_sep
- for k,dof in dofs.iteritems():
+ for k,dof in dofs.items():
start_row += str(dof.getStart()).ljust(col_spacing) + col_sep
f.write(header)
f.write(start_row)
for idx,pt in enumerate(rwgt_pts):
row_name = "rwgt%d" % (idx)
row = "\n" + row_name.ljust(col_spacing) + col_sep
- for k,dof in dofs.iteritems():
- if not pt.has_key(dof.getName()):
+ for k,dof in dofs.items():
+ if dof.getName() not in pt:
row += "0.0".ljust(col_spacing) + col_sep
else:
row += str(pt[dof.getName()]).ljust(col_spacing) + col_sep
From 06958ac0d59e819423a32f8c7d7f1641e7e1c32f Mon Sep 17 00:00:00 2001
From: Andrew Wightman <awightma@login.uscms.org>
Date: Wed, 10 Jul 2024 18:11:16 -0500
Subject: [PATCH 3/4] Convert byte string returned by popen
---
mcgeneration/helpers/helper_tools.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mcgeneration/helpers/helper_tools.py b/mcgeneration/helpers/helper_tools.py
index c72edff..e3d2e65 100644
--- a/mcgeneration/helpers/helper_tools.py
+++ b/mcgeneration/helpers/helper_tools.py
@@ -10,7 +10,7 @@ def run_process(inputs,verbose=True,indent=0):
p = subprocess.Popen(inputs,stdout=subprocess.PIPE)
stdout = []
while True:
- l = p.stdout.readline()
+ l = p.stdout.readline().decode('utf-8')
if l == '' and p.poll() is not None:
break
if l:
From a32d9a777d9d8aebeaef977dcaad854fb8ab56cf Mon Sep 17 00:00:00 2001
From: Andrew42 <awightma@nd.edu>
Date: Wed, 10 Jul 2024 18:14:21 -0500
Subject: [PATCH 4/4] Update proper hostname check for cmsconnect
---
mcgeneration/setup_production.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/mcgeneration/setup_production.sh b/mcgeneration/setup_production.sh
index 2c779fe..f949fa9 100644
--- a/mcgeneration/setup_production.sh
+++ b/mcgeneration/setup_production.sh
@@ -5,8 +5,8 @@
EFTMCPATH=`pwd -P`
# path should end with genproductions
GENPRODPATH=${EFTMCPATH}/../../genproductions
-#if [ "$(hostname)" == "login.uscms.org" ]; then
-if [ "$(hostname)" == "login-el7.uscms.org" ]; then
+if [ "$(hostname)" == "login.uscms.org" ]; then
+#if [ "$(hostname)" == "login-el7.uscms.org" ]; then
# path for cmsconnect submit node
GENPRODPATH="/ospool/cms-user/${USER}/genproductions"
fi