diff --git a/doc/versionHistory.rst b/doc/versionHistory.rst index 756ff18c..6174d181 100644 --- a/doc/versionHistory.rst +++ b/doc/versionHistory.rst @@ -6,6 +6,39 @@ Version History ################## +.. _lsst.ts.wep-13.0.4: + +------------- +13.0.4 +------------- + +* Increased maxFracBadPixels in pipelines to 8 pixels per 200^2. +* Updated configs for TIE and Danish production pipelines to reflect current defaults + +.. _lsst.ts.wep-13.0.3: + +------------- +13.0.3 +------------- + +* Task plotPsfFromZern added in comCamRapidAnalysisPipeline and comCamRapidAnalysisDanishPipeline. + +.. _lsst.ts.wep-13.0.3: + +------------- +13.0.2 +------------- + +* Use _refresh_metadata in cutOutStamps function so DonutStamps have correct set of metadata when running cutOutDonuts tasks interactively. + +.. _lsst.ts.wep-13.0.1: + +------------- +13.0.1 +------------- + +* Reorganize pipelines and add daily processing and danish pipelines. + .. _lsst.ts.wep-13.0.0: ------------- diff --git a/pipelines/_ingredients/donutVizGroupPipeline.yaml b/pipelines/_ingredients/donutVizGroupPipeline.yaml new file mode 100644 index 00000000..96d5c03d --- /dev/null +++ b/pipelines/_ingredients/donutVizGroupPipeline.yaml @@ -0,0 +1,42 @@ +description: donut viz pipeline tasks + +tasks: + aggregateZernikeTablesTask: + class: lsst.donut.viz.AggregateZernikeTablesTask + aggregateDonutTablesGroupTask: + class: lsst.donut.viz.AggregateDonutTablesTask + config: + python: | + from lsst.ts.wep.task.pairTask import GroupPairer + config.pairer.retarget(GroupPairer) + aggregateAOSVisitTableTask: + class: lsst.donut.viz.AggregateAOSVisitTableTask + plotAOSTask: + class: lsst.donut.viz.PlotAOSTask + config: + doRubinTVUpload: true + aggregateDonutStampsTask: + class: lsst.donut.viz.AggregateDonutStampsTask + plotDonutTask: + class: lsst.donut.viz.PlotDonutTask + config: + doRubinTVUpload: true + plotPsfZernTask: + class: lsst.donut.viz.PlotPsfZernTask + config: + doRubinTVUpload: true + +subsets: + donutVizGroups: + subset: + - aggregateZernikeTablesTask + - aggregateDonutTablesGroupTask + - aggregateAOSVisitTableTask + - plotAOSTask + - aggregateDonutStampsTask + - plotDonutTask + - plotPsfZernTask + description: | + AOS Donut visualization plotting tasks. This step generates plots + (including the pyramid residual and donut gallery) and + tables for the AOS visit. diff --git a/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml b/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml new file mode 100644 index 00000000..c02dc4d8 --- /dev/null +++ b/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml @@ -0,0 +1,53 @@ +description: wep direct detect pipeline + +tasks: + generateDonutDirectDetectTask: + class: lsst.ts.wep.task.generateDonutDirectDetectTask.GenerateDonutDirectDetectTask + config: + donutSelector.useCustomMagLimit: True + donutSelector.sourceLimit: 20 + cutOutDonutsScienceSensorGroupTask: + class: lsst.ts.wep.task.cutOutDonutsScienceSensorTask.CutOutDonutsScienceSensorTask + config: + python: | + from lsst.ts.wep.task.pairTask import GroupPairer + config.pairer.retarget(GroupPairer) + donutStampSize: 200 + initialCutoutPadding: 40 + calcZernikesTask: + class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask + config: + estimateZernikes.nollIndices: [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 18, 19, 20, 21, 22, 27, 28] + estimateZernikes.convergeTol: 10.0e-9 + estimateZernikes.requireConverge: True + estimateZernikes.compGain: 0.75 + estimateZernikes.compSequence: [4, 4, 6, 6, 13, 13, 13, 13] + estimateZernikes.maxIter: 50 + estimateZernikes.saveHistory: False + estimateZernikes.maskKwargs: { "doMaskBlends": False } + donutStampSelector.maxSelect: 5 + isr: + class: lsst.ip.isr.IsrTaskLSST + config: + # Although we don't have to apply the amp offset corrections, we do want + # to compute them for analyzeAmpOffsetMetadata to report on as metrics. + doAmpOffset: true + ampOffset.doApplyAmpOffset: false + # Turn off slow steps in ISR + doBrighterFatter: false + # Mask saturated pixels, + # but turn off quadratic crosstalk because it's currently broken + doSaturation: True + crosstalk.doQuadraticCrosstalkCorrection: False + +subsets: + wepDirectDetect: + subset: + - isr + - generateDonutDirectDetectTask + - cutOutDonutsScienceSensorGroupTask + - calcZernikesTask + description: | + This step processes the input images with ISR, + finds and cuts out the donut stamps, + and estimates the Zernike coefficients from the donut pairs. diff --git a/pipelines/directDetect/cwfsDirectDetectPipeline.yaml b/pipelines/directDetect/cwfsDirectDetectPipeline.yaml deleted file mode 100644 index 8487b230..00000000 --- a/pipelines/directDetect/cwfsDirectDetectPipeline.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# This yaml file is used to define the tasks and configuration of -# a Gen 3 pipeline used for testing in ts_wep. -description: wep basic processing test pipeline -# Here we specify the corresponding instrument for the data we -# will be using. -instrument: lsst.obs.lsst.LsstCam -# Then we can specify each task in our pipeline by a name -# and then specify the class name corresponding to that task -tasks: - isr: - class: lsst.ip.isr.isrTask.IsrTask - # Below we specify the configuration settings we want to use - # when running the task in this pipeline. Since our data doesn't - # include bias or flats we only want to use doApplyGains and - # doOverscan in our isr task. - config: - connections.outputExposure: "postISRCCD" - doBias: False - doVariance: False - doLinearize: False - doCrosstalk: False - doDefect: False - doNanMasking: False - doInterpolate: False - doBrighterFatter: False - doDark: False - doFlat: False - doApplyGains: True - doFringe: False - doOverscan: True - python: OverscanCorrectionTask.ConfigClass.fitType = 'MEDIAN' - generateDonutDirectDetectTask: - class: lsst.ts.wep.task.generateDonutDirectDetectTask.GenerateDonutDirectDetectTask - config: - donutSelector.useCustomMagLimit: True - cutOutDonutsCwfsTask: - class: lsst.ts.wep.task.cutOutDonutsCwfsTask.CutOutDonutsCwfsTask - calcZernikesTask: - class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask - config: - estimateZernikes.maxNollIndex: 28 - estimateZernikes.saveHistory: False - estimateZernikes.maskKwargs: {'doMaskBlends': False} diff --git a/pipelines/directDetect/scienceDirectDetectPipeline.yaml b/pipelines/directDetect/scienceDirectDetectPipeline.yaml deleted file mode 100644 index d82432d6..00000000 --- a/pipelines/directDetect/scienceDirectDetectPipeline.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# This yaml file is used to define the tasks and configuration of -# a Gen 3 pipeline used for testing in ts_wep. -description: wep basic processing test pipeline -# Here we specify the corresponding instrument for the data we -# will be using. -instrument: lsst.obs.lsst.LsstCam -# Then we can specify each task in our pipeline by a name -# and then specify the class name corresponding to that task -tasks: - isr: - class: lsst.ip.isr.isrTask.IsrTask - # Below we specify the configuration settings we want to use - # when running the task in this pipeline. Since our data doesn't - # include bias or flats we only want to use doApplyGains and - # doOverscan in our isr task. - config: - connections.outputExposure: "postISRCCD" - doBias: False - doVariance: False - doLinearize: False - doCrosstalk: False - doDefect: False - doNanMasking: False - doInterpolate: False - doBrighterFatter: False - doDark: False - doFlat: False - doApplyGains: True - doFringe: False - doOverscan: True - python: OverscanCorrectionTask.ConfigClass.fitType = 'MEDIAN' - generateDonutDirectDetectTask: - class: lsst.ts.wep.task.generateDonutDirectDetectTask.GenerateDonutDirectDetectTask - config: - donutSelector.useCustomMagLimit: True - cutOutDonutsScienceSensorTask: - class: lsst.ts.wep.task.CutOutDonutsScienceSensorTask - calcZernikesTask: - class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask - config: - estimateZernikes.maxNollIndex: 28 - estimateZernikes.saveHistory: False - estimateZernikes.maskKwargs: {'doMaskBlends': False} diff --git a/pipelines/production/comCamDailyProcessing.yaml b/pipelines/production/comCamDailyProcessing.yaml new file mode 100644 index 00000000..ee208b76 --- /dev/null +++ b/pipelines/production/comCamDailyProcessing.yaml @@ -0,0 +1,46 @@ +description: Run WEP + DonutViz daily at USDF +instrument: lsst.obs.lsst.LsstComCam +imports: + - $TS_WEP_DIR/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml + - $TS_WEP_DIR/pipelines/_ingredients/donutVizGroupPipeline.yaml + +tasks: + generateDonutDirectDetectTask: + class: lsst.ts.wep.task.generateDonutDirectDetectTask.GenerateDonutDirectDetectTask + config: + donutSelector.useCustomMagLimit: True + plotAOSTask: + class: lsst.donut.viz.PlotAOSTask + config: + doRubinTVUpload: false + aggregateDonutStampsTask: + class: lsst.donut.viz.AggregateDonutStampsTask + plotDonutTask: + class: lsst.donut.viz.PlotDonutTask + config: + doRubinTVUpload: false + +# Define pipeline steps +subsets: + step1: + subset: + - isr + - generateDonutDirectDetectTask + - cutOutDonutsScienceSensorGroupTask + - calcZernikesTask + description: | + This step processes the input images with ISR, + finds and cuts out the donut stamps, + and estimates the Zernike coefficients from the donut pairs. + step2a: + subset: + - aggregateZernikeTablesTask + - aggregateDonutTablesGroupTask + - aggregateAOSVisitTableTask + - plotAOSTask + - aggregateDonutStampsTask + - plotDonutTask + description: | + AOS Donut visualization plotting tasks. This step generates plots + (including the pyramid residual and donut gallery) and + tables for the AOS visit. diff --git a/pipelines/production/comCamRapidAnalysisDanishPipeline.yaml b/pipelines/production/comCamRapidAnalysisDanishPipeline.yaml new file mode 100644 index 00000000..9763b4bf --- /dev/null +++ b/pipelines/production/comCamRapidAnalysisDanishPipeline.yaml @@ -0,0 +1,44 @@ +description: rapid analysis pipeline for ComCam w/ danish zernike estimation +instrument: lsst.obs.lsst.LsstComCam +imports: + - $TS_WEP_DIR/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml + - $TS_WEP_DIR/pipelines/_ingredients/donutVizGroupPipeline.yaml + +tasks: + calcZernikesTask: + class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask + config: + python: | + from lsst.ts.wep.task import EstimateZernikesDanishTask + config.estimateZernikes.retarget(EstimateZernikesDanishTask) + donutStampSelector.maxSelect: 5 + donutStampSelector.maxFracBadPixels: 2.0e-4 + estimateZernikes.binning: 2 + estimateZernikes.nollIndices: + [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 20, 21, 22, 27, 28] + +# Define pipeline steps +subsets: + step1: + subset: + - isr + - generateDonutDirectDetectTask + - cutOutDonutsScienceSensorGroupTask + - calcZernikesTask + description: | + This step processes the input images with ISR, + finds and cuts out the donut stamps, + and estimates the Zernike coefficients from the donut pairs. + step2a: + subset: + - aggregateZernikeTablesTask + - aggregateDonutTablesGroupTask + - aggregateAOSVisitTableTask + - plotAOSTask + - aggregateDonutStampsTask + - plotDonutTask + - plotPsfZernTask + description: | + AOS Donut visualization plotting tasks. This step generates plots + (including the pyramid residual and donut gallery) and + tables for the AOS visit. diff --git a/pipelines/production/comCamRapidAnalysisPipeline.yaml b/pipelines/production/comCamRapidAnalysisPipeline.yaml index 9d13df60..5f86060a 100644 --- a/pipelines/production/comCamRapidAnalysisPipeline.yaml +++ b/pipelines/production/comCamRapidAnalysisPipeline.yaml @@ -1,61 +1,23 @@ description: rapid analysis pipeline for ComCam instrument: lsst.obs.lsst.LsstComCam +imports: + - $TS_WEP_DIR/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml + - $TS_WEP_DIR/pipelines/_ingredients/donutVizGroupPipeline.yaml + tasks: - generateDonutDirectDetectTask: - class: lsst.ts.wep.task.generateDonutDirectDetectTask.GenerateDonutDirectDetectTask - config: - donutSelector.useCustomMagLimit: True - donutSelector.sourceLimit: 20 - cutOutDonutsScienceSensorGroupTask: - class: lsst.ts.wep.task.cutOutDonutsScienceSensorTask.CutOutDonutsScienceSensorTask - config: - python: | - from lsst.ts.wep.task.pairTask import GroupPairer - config.pairer.retarget(GroupPairer) - donutStampSize: 200 - initialCutoutPadding: 40 calcZernikesTask: class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask config: - estimateZernikes.nollIndices: [4, 5, 6, 7, 8, 9, 10, 11, 14, 15, 20, 21, 27, 28] + estimateZernikes.nollIndices: [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 18, 19, 20, 21, 22, 27, 28] estimateZernikes.convergeTol: 10.0e-9 + estimateZernikes.compGain: 0.75 + estimateZernikes.compSequence: [4, 4, 6, 6, 13, 13, 13, 13] + estimateZernikes.maxIter: 50 estimateZernikes.requireConverge: True estimateZernikes.saveHistory: False estimateZernikes.maskKwargs: { "doMaskBlends": False } donutStampSelector.maxSelect: 5 - isr: - class: lsst.ip.isr.IsrTaskLSST - config: - # Although we don't have to apply the amp offset corrections, we do want - # to compute them for analyzeAmpOffsetMetadata to report on as metrics. - doAmpOffset: true - ampOffset.doApplyAmpOffset: false - # Turn off slow steps in ISR - doBrighterFatter: false - # Mask saturated pixels, - # but turn off quadratic crosstalk because it's currently broken - doSaturation: True - crosstalk.doQuadraticCrosstalkCorrection: False - aggregateZernikeTablesTask: - class: lsst.donut.viz.AggregateZernikeTablesTask - aggregateDonutTablesGroupTask: - class: lsst.donut.viz.AggregateDonutTablesTask - config: - python: | - from lsst.ts.wep.task.pairTask import GroupPairer - config.pairer.retarget(GroupPairer) - aggregateAOSVisitTableTask: - class: lsst.donut.viz.AggregateAOSVisitTableTask - plotAOSTask: - class: lsst.donut.viz.PlotAOSTask - config: - doRubinTVUpload: true - aggregateDonutStampsTask: - class: lsst.donut.viz.AggregateDonutStampsTask - plotDonutTask: - class: lsst.donut.viz.PlotDonutTask - config: - doRubinTVUpload: true + donutStampSelector.maxFracBadPixels: 2.0e-4 # Define pipeline steps subsets: @@ -77,6 +39,7 @@ subsets: - plotAOSTask - aggregateDonutStampsTask - plotDonutTask + - plotPsfZernTask description: | AOS Donut visualization plotting tasks. This step generates plots (including the pyramid residual and donut gallery) and diff --git a/pipelines/production/comCamSimRapidAnalysisPipeline.yaml b/pipelines/production/comCamSimRapidAnalysisPipeline.yaml index 9be9f33c..6bbda11a 100644 --- a/pipelines/production/comCamSimRapidAnalysisPipeline.yaml +++ b/pipelines/production/comCamSimRapidAnalysisPipeline.yaml @@ -1,54 +1,8 @@ description: rapid analysis pipeline for ComCamSim instrument: lsst.obs.lsst.LsstComCamSim -tasks: - generateDonutDirectDetectTask: - class: lsst.ts.wep.task.generateDonutDirectDetectTask.GenerateDonutDirectDetectTask - config: - donutSelector.useCustomMagLimit: True - donutSelector.sourceLimit: 20 - cutOutDonutsScienceSensorGroupTask: - class: lsst.ts.wep.task.cutOutDonutsScienceSensorTask.CutOutDonutsScienceSensorTask - config: - python: | - from lsst.ts.wep.task.pairTask import GroupPairer - config.pairer.retarget(GroupPairer) - donutStampSize: 200 - initialCutoutPadding: 40 - calcZernikesTask: - class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask - config: - estimateZernikes.nollIndices: [4, 5, 6, 7, 8, 9, 10, 11, 14, 15, 20, 21, 27, 28] - estimateZernikes.convergeTol: 10.0e-9 - estimateZernikes.requireConverge: True - estimateZernikes.saveHistory: False - estimateZernikes.maskKwargs: { "doMaskBlends": False } - donutStampSelector.maxSelect: 5 - isr: - class: lsst.ip.isr.IsrTask - config: - # Although we don't have to apply the amp offset corrections, we do want - # to compute them for analyzeAmpOffsetMetadata to report on as metrics. - doAmpOffset: true - ampOffset.doApplyAmpOffset: false - # Turn off slow steps in ISR - doBrighterFatter: false - doCrosstalk: false - aggregateZernikeTablesTask: - class: lsst.donut.viz.AggregateZernikeTablesTask - aggregateDonutTablesTask: - class: lsst.donut.viz.AggregateDonutTablesTask - aggregateAOSVisitTableTask: - class: lsst.donut.viz.AggregateAOSVisitTableTask - plotAOSTask: - class: lsst.donut.viz.PlotAOSTask - config: - doRubinTVUpload: true - aggregateDonutStampsTask: - class: lsst.donut.viz.AggregateDonutStampsTask - plotDonutTask: - class: lsst.donut.viz.PlotDonutTask - config: - doRubinTVUpload: true +imports: + - $TS_WEP_DIR/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml + - $TS_WEP_DIR/pipelines/_ingredients/donutVizGroupPipeline.yaml # Define pipeline steps subsets: diff --git a/python/lsst/ts/wep/task/cutOutDonutsBase.py b/python/lsst/ts/wep/task/cutOutDonutsBase.py index 6976035c..e835090e 100644 --- a/python/lsst/ts/wep/task/cutOutDonutsBase.py +++ b/python/lsst/ts/wep/task/cutOutDonutsBase.py @@ -690,20 +690,10 @@ def cutOutStamps(self, exposure, donutCatalog, defocalType, cameraName): finalStamps.append(donutStamp) - # Calculate the difference between original centroid and final centroid + # Add additional information into metadata catalogLength = len(donutCatalog) stampsMetadata = PropertyList() - stampsMetadata["RA_DEG"] = np.degrees(donutCatalog["coord_ra"].value) - stampsMetadata["DEC_DEG"] = np.degrees(donutCatalog["coord_dec"].value) - stampsMetadata["DET_NAME"] = np.array([detectorName] * catalogLength, dtype=str) - stampsMetadata["CAM_NAME"] = np.array([cameraName] * catalogLength, dtype=str) stampsMetadata["VISIT"] = np.array([visitId] * catalogLength, dtype=int) - stampsMetadata["DFC_TYPE"] = np.array( - [defocalType.value] * catalogLength, dtype=str - ) - stampsMetadata["DFC_DIST"] = np.array( - [instrument.defocalOffset * 1e3] * catalogLength - ) # Save the donut flux as magnitude if len(donutCatalog["source_flux"]) > 0: stampsMetadata["MAG"] = ( @@ -714,21 +704,12 @@ def cutOutStamps(self, exposure, donutCatalog, defocalType, cameraName): # Save the original centroid values stampsMetadata["CENT_X0"] = np.array(donutCatalog["centroid_x"].value) stampsMetadata["CENT_Y0"] = np.array(donutCatalog["centroid_y"].value) - # Save the centroid values - stampsMetadata["CENT_X"] = donutCatalog["finalDonutX"] - stampsMetadata["CENT_Y"] = donutCatalog["finalDonutY"] # Save the centroid shift stampsMetadata["CENT_DX"] = donutCatalog["xShift"] stampsMetadata["CENT_DY"] = donutCatalog["yShift"] stampsMetadata["CENT_DR"] = np.sqrt( donutCatalog["xShift"] ** 2 + donutCatalog["yShift"] ** 2 ) - # Save the centroid positions of blended sources - stampsMetadata["BLEND_CX"] = np.array(finalBlendXList, dtype=str) - stampsMetadata["BLEND_CY"] = np.array(finalBlendYList, dtype=str) - # Save the corner values - stampsMetadata["X0"] = donutCatalog["xCorner"] - stampsMetadata["Y0"] = donutCatalog["yCorner"] # Save visit info stampsMetadata["BORESIGHT_ROT_ANGLE_RAD"] = ( donutCatalog.meta["visit_info"]["boresight_rot_angle"].to(u.rad).value @@ -801,4 +782,11 @@ def cutOutStamps(self, exposure, donutCatalog, defocalType, cameraName): # Save the fraction of bad pixels stampsMetadata["FRAC_BAD_PIX"] = np.array(fracBadPixels).astype(float) - return DonutStamps(finalStamps, metadata=stampsMetadata, use_archive=True) + finalDonutStamps = DonutStamps( + finalStamps, metadata=stampsMetadata, use_archive=True + ) + # Refresh to pull original metadata into stamps + # Necessary when running full pipeline interactively. + finalDonutStamps._refresh_metadata() + + return finalDonutStamps diff --git a/tests/task/test_cutOutDonutsBase.py b/tests/task/test_cutOutDonutsBase.py index 541f9594..47e7e19b 100644 --- a/tests/task/test_cutOutDonutsBase.py +++ b/tests/task/test_cutOutDonutsBase.py @@ -430,6 +430,7 @@ def testCutOutStampsTaskRunNormal(self): "BORESIGHT_AZ_RAD", "BORESIGHT_RA_RAD", "BORESIGHT_DEC_RAD", + "BANDPASS", ] self.assertCountEqual(metadata, expectedMetadata) diff --git a/tests/task/test_cutOutDonutsScienceSensorTask.py b/tests/task/test_cutOutDonutsScienceSensorTask.py index f80e278b..6c9d6755 100644 --- a/tests/task/test_cutOutDonutsScienceSensorTask.py +++ b/tests/task/test_cutOutDonutsScienceSensorTask.py @@ -310,6 +310,7 @@ def testTaskRun(self): "BORESIGHT_AZ_RAD", "BORESIGHT_RA_RAD", "BORESIGHT_DEC_RAD", + "BANDPASS", ] # Test that all expected metadata is included in the butler self.assertEqual(