diff --git a/FWCore/Framework/src/ScheduleItems.cc b/FWCore/Framework/src/ScheduleItems.cc index e2c23426e8f5f..008cb4a1041c0 100644 --- a/FWCore/Framework/src/ScheduleItems.cc +++ b/FWCore/Framework/src/ScheduleItems.cc @@ -134,8 +134,15 @@ namespace edm { act_table_ = std::make_unique(parameterSet); std::string processName = parameterSet.getParameter("@process_name"); - processConfiguration_ = std::make_shared( - processName, getReleaseVersion(), getPassID()); // propagate_const has no reset() function + std::string releaseVersion; + if (parameterSet.existsAs("@special_override_release_version_only_for_testing", false)) { + releaseVersion = + parameterSet.getUntrackedParameter("@special_override_release_version_only_for_testing"); + } else { + releaseVersion = getReleaseVersion(); + } + // propagate_const has no reset() function + processConfiguration_ = std::make_shared(processName, releaseVersion, getPassID()); auto common = std::make_shared( parameterSet.getUntrackedParameterSet("maxEvents").getUntrackedParameter("input"), parameterSet.getUntrackedParameterSet("maxLuminosityBlocks").getUntrackedParameter("input"), diff --git a/FWCore/ParameterSet/python/Config.py b/FWCore/ParameterSet/python/Config.py index f8c881389b423..daf766cd638d1 100644 --- a/FWCore/ParameterSet/python/Config.py +++ b/FWCore/ParameterSet/python/Config.py @@ -149,6 +149,7 @@ def __init__(self,name: str, *Mods): self.__isStrict = False self.__dict__['_Process__modifiers'] = Mods self.__dict__['_Process__accelerators'] = {} + self.__dict__['_Process__specialOverrideReleaseVersionOnlyForTesting'] = None self.__injectValidValue('options', Process.defaultOptions_()) self.__injectValidValue('maxEvents', Process.defaultMaxEvents_()) self.maxLuminosityBlocks = Process.defaultMaxLuminosityBlocks_() @@ -813,6 +814,10 @@ def extend(self,other,items=()): self.__dict__['_Process__InExtendCall'] = False + def _specialOverrideReleaseVersionOnlyForTesting(self, version): + "This function is intended only for specific framework tests. Do not use for anything else." + self.__specialOverrideReleaseVersionOnlyForTesting = version + def _dumpConfigNamedList(self,items,typeName:str,options:PrintOptions) -> str: returnValue = '' for name,item in items: @@ -1458,6 +1463,8 @@ def __extractPSet(self,pset): self.validate() processPSet.addString(True, "@process_name", self.name_()) + if self.__specialOverrideReleaseVersionOnlyForTesting is not None: + processPSet.addString(False, "@special_override_release_version_only_for_testing", self.__specialOverrideReleaseVersionOnlyForTesting) self.handleProcessAccelerators(processPSet) all_modules = self.producers_().copy() all_modules.update(self.filters_()) @@ -4944,4 +4951,16 @@ def testProcessAccelerator(self): self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"]) self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1]) + def testProcessSpecialOverrideReleaseVersion(self): + proc = Process("TEST") + p = TestMakePSet() + proc.fillProcessDesc(p) + self.assertFalse("@special_override_release_version_only_for_testing" in p.values) + + proc = Process("TEST") + proc._specialOverrideReleaseVersionOnlyForTesting("CMSSW_15_0_0") + p = TestMakePSet() + proc.fillProcessDesc(p) + self.assertEqual((False, "CMSSW_15_0_0"), p.values["@special_override_release_version_only_for_testing"]) + unittest.main() diff --git a/IOPool/Input/test/BuildFile.xml b/IOPool/Input/test/BuildFile.xml index d6d7c23161096..7aeda0519ec7c 100644 --- a/IOPool/Input/test/BuildFile.xml +++ b/IOPool/Input/test/BuildFile.xml @@ -38,5 +38,6 @@ + diff --git a/IOPool/Input/test/testReducedProcessHistory.sh b/IOPool/Input/test/testReducedProcessHistory.sh new file mode 100755 index 0000000000000..6000c5de65653 --- /dev/null +++ b/IOPool/Input/test/testReducedProcessHistory.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +function die { echo $1: status $2 ; exit $2; } +function runSuccess { + echo "cmsRun $@" + cmsRun $@ || die "cmsRun $*" $? + echo +} +function runFailure { + echo "cmsRun $@ (expected to fail)" + cmsRun $@ && die "cmsRun $*" 1 + echo +} + +VERSION_ARR=(${CMSSW_VERSION//_/ }) +VERSION1="${VERSION_ARR[0]}_${VERSION_ARR[1]}_${VERSION_ARR[2]}_0" +VERSION2="${VERSION_ARR[0]}_${VERSION_ARR[1]}_${VERSION_ARR[2]}_1" +VERSION3="${VERSION_ARR[0]}_${VERSION_ARR[1]}_$((${VERSION_ARR[2]}+1))_0" + +# Check that changing the patch version does not lead to new lumi or run +runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --version ${VERSION1} --firstEvent 1 --output version1.root +runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --version ${VERSION2} --firstEvent 101 --output version2.root + +edmProvDump version1.root | grep -q "PROD.*'${VERSION1}'" || die "Did not find ${VERSION1} from version.root provenance" $? +edmProvDump version2.root | grep -q "PROD.*'${VERSION2}'" || die "Did not find ${VERSION2} from version.root provenance" $? + +runSuccess ${SCRAM_TEST_PATH}/test_merge_two_files.py version1.root version2.root + +runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files.root + + +# Check that changing the minor version leads to new lumi +runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --version ${VERSION3} --firstEvent 201 --output version3_lumi.root + +edmProvDump version3_lumi.root | grep -q "PROD.*'${VERSION3}'" || die "Did not find ${VERSION3} from version3_lumi.root provenance" $? + +runSuccess ${SCRAM_TEST_PATH}/test_merge_two_files.py version1.root version3_lumi.root --output merged_files3_lumi.root --bypassVersionCheck + +runFailure ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files3_lumi.root --bypassVersionCheck + +runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files3_lumi.root --bypassVersionCheck --expectNewLumi + + +# Check that changing the minor version leads to new run +runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --version ${VERSION3} --firstEvent 201 --lumi 2 --output version3_run.root + +edmProvDump version3_run.root | grep -q "PROD.*'${VERSION3}'" || die "Did not find ${VERSION3} from version3_lumi.root provenance" $? + +runSuccess ${SCRAM_TEST_PATH}/test_merge_two_files.py version1.root version3_run.root --output merged_files3_run.root --bypassVersionCheck + +runFailure ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files3_run.root --bypassVersionCheck + +runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files3_run.root --bypassVersionCheck --expectNewRun + +exit 0 diff --git a/IOPool/Input/test/testReducedProcessHistoryCreate_cfg.py b/IOPool/Input/test/testReducedProcessHistoryCreate_cfg.py new file mode 100644 index 0000000000000..ca896235e7fc0 --- /dev/null +++ b/IOPool/Input/test/testReducedProcessHistoryCreate_cfg.py @@ -0,0 +1,39 @@ +import FWCore.ParameterSet.Config as cms +import argparse + +parser = argparse.ArgumentParser(description='Create files for reduced ProcessHistory test') +parser.add_argument("--version", type=str, help="CMSSW version to be used in the ProcessHistory") +parser.add_argument("--firstEvent", default=1, type=int, help="Number of first event") +parser.add_argument("--lumi", default=1, type=int, help="LuminosityBlock number") +parser.add_argument("--output", type=str, help="Output file name") + +args = parser.parse_args() + +process = cms.Process("PROD") +process._specialOverrideReleaseVersionOnlyForTesting(args.version) + +process.maxEvents.input = 10 + +from FWCore.Modules.modules import EmptySource +process.source = EmptySource( + firstEvent = args.firstEvent, + firstLuminosityBlock = args.lumi, +) + +from IOPool.Output.modules import PoolOutputModule +process.out = PoolOutputModule( + fileName = args.output +) + +from FWCore.Framework.modules import IntProducer +process.intProducer = IntProducer(ivalue = 42) + +from FWCore.Integration.modules import ThingWithMergeProducer +process.thingWithMergeProducer = ThingWithMergeProducer() + +process.t = cms.Task( + process.intProducer, + process.thingWithMergeProducer, +) +process.p = cms.Path(process.t) +process.ep = cms.EndPath(process.out) diff --git a/IOPool/Input/test/testReducedProcessHistory_cfg.py b/IOPool/Input/test/testReducedProcessHistory_cfg.py new file mode 100644 index 0000000000000..4d4ff297670f5 --- /dev/null +++ b/IOPool/Input/test/testReducedProcessHistory_cfg.py @@ -0,0 +1,117 @@ +import FWCore.ParameterSet.Config as cms +import argparse + +parser = argparse.ArgumentParser(description='Test reduced ProcessHistory') +parser.add_argument("--input", type=str, help="Input file") +parser.add_argument("--bypassVersionCheck", action="store_true", help="Bypass version check") +parser.add_argument("--expectNewLumi", action="store_true", help="Set this if a new lumi is expected between the original files") +parser.add_argument("--expectNewRun", action="store_true", help="Set this if a new run is expected between the original files") + +args = parser.parse_args() + +process = cms.Process("READ") + +from IOPool.Input.modules import PoolSource +process.source = PoolSource( + fileNames = [f"file:{args.input}"], + bypassVersionCheck = args.bypassVersionCheck, +) + +from FWCore.Framework.modules import TestMergeResults, RunLumiEventAnalyzer +process.testmerge = TestMergeResults( + expectedBeginRunProd = [10001, 20004, 10003], + expectedEndRunProd = [100001, 200004, 100003], + expectedBeginLumiProd = [101, 204, 103], + expectedEndLumiProd = [1001, 2004, 1003], + + expectedBeginRunNew = [10001, 10002, 10003], + expectedEndRunNew = [100001, 100002, 100003], + expectedBeginLumiNew = [101, 102, 103], + expectedEndLumiNew = [1001, 1002, 1003], + expectedProcessHistoryInRuns = [ + 'PROD', + 'MERGETWOFILES', + 'READ' + ] +) +def setWithMergeAndCopyEntry(p, value): + p[1] = value + p[3:5] = p[0:2] +if args.expectNewLumi or args.expectNewRun: + setWithMergeAndCopyEntry(process.testmerge.expectedBeginRunProd, 10002) + setWithMergeAndCopyEntry(process.testmerge.expectedEndRunProd, 100002) + setWithMergeAndCopyEntry(process.testmerge.expectedBeginLumiProd, 102) + setWithMergeAndCopyEntry(process.testmerge.expectedEndLumiProd, 1002) + +process.test = RunLumiEventAnalyzer( + expectedRunLumiEvents = [ + 1, 0, 0, # beginRun + 1, 1, 0, # beginLumi + 1, 1, 1, + 1, 1, 2, + 1, 1, 3, + 1, 1, 4, + 1, 1, 5, + 1, 1, 6, + 1, 1, 7, + 1, 1, 8, + 1, 1, 9, + 1, 1, 10, + 1, 1, 101, + 1, 1, 102, + 1, 1, 103, + 1, 1, 104, + 1, 1, 105, + 1, 1, 106, + 1, 1, 107, + 1, 1, 108, + 1, 1, 109, + 1, 1, 110, + 1, 1, 0, # endLumi + 1, 0, 0, # endRun + ] +) +endFirstFileIndex = 3*(10+2) +if args.expectNewLumi: + process.test.expectedRunLumiEvents = process.test.expectedRunLumiEvents[:endFirstFileIndex] + [ + 1, 1, 0, # endLumi + 1, 0, 0, # endRun + 1, 0, 0, # beginRun + 1, 1, 0, # beginLumi + 1, 1, 201, + 1, 1, 202, + 1, 1, 203, + 1, 1, 204, + 1, 1, 205, + 1, 1, 206, + 1, 1, 207, + 1, 1, 208, + 1, 1, 209, + 1, 1, 210, + 1, 1, 0, # endLumi + 1, 0, 0, # endRun + ] +elif args.expectNewRun: + process.test.expectedRunLumiEvents = process.test.expectedRunLumiEvents[:endFirstFileIndex] + [ + 1, 1, 0, # endLumi + 1, 0, 0, # endRun + 1, 0, 0, # beginRun + 1, 2, 0, # beginLumi + 1, 2, 201, + 1, 2, 202, + 1, 2, 203, + 1, 2, 204, + 1, 2, 205, + 1, 2, 206, + 1, 2, 207, + 1, 2, 208, + 1, 2, 209, + 1, 2, 210, + 1, 2, 0, # endLumi + 1, 0, 0, # endRun + ] + +process.p = cms.Path( + process.testmerge + + process.test +) diff --git a/IOPool/Input/test/test_merge_two_files.py b/IOPool/Input/test/test_merge_two_files.py index e12272830dd50..043287436b650 100644 --- a/IOPool/Input/test/test_merge_two_files.py +++ b/IOPool/Input/test/test_merge_two_files.py @@ -1,18 +1,28 @@ import FWCore.ParameterSet.Config as cms -import sys +import argparse + +parser = argparse.ArgumentParser(description='Create files for reduced ProcessHistory test') +parser.add_argument("file1", type=str, help="First file to merge") +parser.add_argument("file2", type=str, help="Second file to merge") +parser.add_argument("--output", default="merged_files.root", help="Output file name") +parser.add_argument("--bypassVersionCheck", action="store_true", help="Bypass version check") + +args = parser.parse_args() process = cms.Process("MERGETWOFILES") -process.source = cms.Source("PoolSource", - fileNames = cms.untracked.vstring("file:"+sys.argv[1], - "file:"+sys.argv[2]), - duplicateCheckMode = cms.untracked.string("noDuplicateCheck") +from IOPool.Input.modules import PoolSource +process.source = PoolSource( + fileNames = ["file:"+args.file1,"file:"+args.file2], + duplicateCheckMode = "noDuplicateCheck", + bypassVersionCheck = args.bypassVersionCheck, ) -process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") +from FWCore.Integration.modules import ThingWithMergeProducer +process.thingWithMergeProducer = ThingWithMergeProducer() -process.out = cms.OutputModule("PoolOutputModule", - fileName = cms.untracked.string("merged_files.root")) +from IOPool.Output.modules import PoolOutputModule +process.out = PoolOutputModule(fileName = args.output) process.p = cms.Path(process.thingWithMergeProducer)