Merge develop into main

doc/versionHistory.rst

@@ -6,6 +6,39 @@
 Version History
 ##################
 
+.. _lsst.ts.wep-13.0.4:
+
+-------------
+13.0.4
+-------------
+
+* Increased maxFracBadPixels in pipelines to 8 pixels per 200^2.
+* Updated configs for TIE and Danish production pipelines to reflect current defaults
+
+.. _lsst.ts.wep-13.0.3:
+
+-------------
+13.0.3
+-------------
+
+* Task plotPsfFromZern added in comCamRapidAnalysisPipeline and comCamRapidAnalysisDanishPipeline.
+
+.. _lsst.ts.wep-13.0.3:
+
+-------------
+13.0.2
+-------------
+
+* Use _refresh_metadata in cutOutStamps function so DonutStamps have correct set of metadata when running cutOutDonuts tasks interactively.
+
+.. _lsst.ts.wep-13.0.1:
+
+-------------
+13.0.1
+-------------
+
+* Reorganize pipelines and add daily processing and danish pipelines.
+
 .. _lsst.ts.wep-13.0.0:
 
 -------------

pipelines/_ingredients/donutVizGroupPipeline.yaml

@@ -0,0 +1,42 @@
+description: donut viz pipeline tasks
+
+tasks:
+  aggregateZernikeTablesTask:
+    class: lsst.donut.viz.AggregateZernikeTablesTask
+  aggregateDonutTablesGroupTask:
+      class: lsst.donut.viz.AggregateDonutTablesTask
+      config:
+          python: |
+              from lsst.ts.wep.task.pairTask import GroupPairer
+              config.pairer.retarget(GroupPairer)
+  aggregateAOSVisitTableTask:
+    class: lsst.donut.viz.AggregateAOSVisitTableTask
+  plotAOSTask:
+    class: lsst.donut.viz.PlotAOSTask
+    config:
+      doRubinTVUpload: true
+  aggregateDonutStampsTask:
+    class: lsst.donut.viz.AggregateDonutStampsTask
+  plotDonutTask:
+    class: lsst.donut.viz.PlotDonutTask
+    config:
+      doRubinTVUpload: true
+  plotPsfZernTask:
+    class: lsst.donut.viz.PlotPsfZernTask
+    config:
+      doRubinTVUpload: true
+
+subsets:
+  donutVizGroups:
+    subset:
+      - aggregateZernikeTablesTask
+      - aggregateDonutTablesGroupTask
+      - aggregateAOSVisitTableTask
+      - plotAOSTask
+      - aggregateDonutStampsTask
+      - plotDonutTask
+      - plotPsfZernTask
+    description: |
+      AOS Donut visualization plotting tasks. This step generates plots
+      (including the pyramid residual and donut gallery) and
+      tables for the AOS visit.

pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml

@@ -0,0 +1,53 @@
+description: wep direct detect pipeline
+
+tasks:
+  generateDonutDirectDetectTask:
+    class: lsst.ts.wep.task.generateDonutDirectDetectTask.GenerateDonutDirectDetectTask
+    config:
+      donutSelector.useCustomMagLimit: True
+      donutSelector.sourceLimit: 20
+  cutOutDonutsScienceSensorGroupTask:
+    class: lsst.ts.wep.task.cutOutDonutsScienceSensorTask.CutOutDonutsScienceSensorTask
+    config:
+      python: |
+        from lsst.ts.wep.task.pairTask import GroupPairer
+        config.pairer.retarget(GroupPairer)
+      donutStampSize: 200
+      initialCutoutPadding: 40
+  calcZernikesTask:
+    class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask
+    config:
+      estimateZernikes.nollIndices: [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 18, 19, 20, 21, 22, 27, 28]
+      estimateZernikes.convergeTol: 10.0e-9
+      estimateZernikes.requireConverge: True
+      estimateZernikes.compGain: 0.75
+      estimateZernikes.compSequence: [4, 4, 6, 6, 13, 13, 13, 13]
+      estimateZernikes.maxIter: 50
+      estimateZernikes.saveHistory: False
+      estimateZernikes.maskKwargs: { "doMaskBlends": False }
+      donutStampSelector.maxSelect: 5
+  isr:
+    class: lsst.ip.isr.IsrTaskLSST
+    config:
+      # Although we don't have to apply the amp offset corrections, we do want
+      # to compute them for analyzeAmpOffsetMetadata to report on as metrics.
+      doAmpOffset: true
+      ampOffset.doApplyAmpOffset: false
+      # Turn off slow steps in ISR
+      doBrighterFatter: false
+      # Mask saturated pixels,
+      # but turn off quadratic crosstalk because it's currently broken
+      doSaturation: True
+      crosstalk.doQuadraticCrosstalkCorrection: False
+
+subsets:
+  wepDirectDetect:
+    subset:
+      - isr
+      - generateDonutDirectDetectTask
+      - cutOutDonutsScienceSensorGroupTask
+      - calcZernikesTask
+    description: |
+      This step processes the input images with ISR,
+      finds and cuts out the donut stamps,
+      and estimates the Zernike coefficients from the donut pairs.

pipelines/production/comCamDailyProcessing.yaml

@@ -0,0 +1,46 @@
+description: Run WEP + DonutViz daily at USDF
+instrument: lsst.obs.lsst.LsstComCam
+imports:
+  - $TS_WEP_DIR/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml
+  - $TS_WEP_DIR/pipelines/_ingredients/donutVizGroupPipeline.yaml
+
+tasks:
+  generateDonutDirectDetectTask:
+    class: lsst.ts.wep.task.generateDonutDirectDetectTask.GenerateDonutDirectDetectTask
+    config:
+      donutSelector.useCustomMagLimit: True
+  plotAOSTask:
+    class: lsst.donut.viz.PlotAOSTask
+    config:
+      doRubinTVUpload: false
+  aggregateDonutStampsTask:
+    class: lsst.donut.viz.AggregateDonutStampsTask
+  plotDonutTask:
+    class: lsst.donut.viz.PlotDonutTask
+    config:
+      doRubinTVUpload: false
+
+# Define pipeline steps
+subsets:
+  step1:
+    subset:
+      - isr
+      - generateDonutDirectDetectTask
+      - cutOutDonutsScienceSensorGroupTask
+      - calcZernikesTask
+    description: |
+      This step processes the input images with ISR,
+      finds and cuts out the donut stamps,
+      and estimates the Zernike coefficients from the donut pairs.
+  step2a:
+    subset:
+      - aggregateZernikeTablesTask
+      - aggregateDonutTablesGroupTask
+      - aggregateAOSVisitTableTask
+      - plotAOSTask
+      - aggregateDonutStampsTask
+      - plotDonutTask
+    description: |
+      AOS Donut visualization plotting tasks. This step generates plots
+      (including the pyramid residual and donut gallery) and
+      tables for the AOS visit.

pipelines/production/comCamRapidAnalysisDanishPipeline.yaml

@@ -0,0 +1,44 @@
+description: rapid analysis pipeline for ComCam w/ danish zernike estimation
+instrument: lsst.obs.lsst.LsstComCam
+imports:
+  - $TS_WEP_DIR/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml
+  - $TS_WEP_DIR/pipelines/_ingredients/donutVizGroupPipeline.yaml
+
+tasks:
+  calcZernikesTask:
+    class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask
+    config:
+      python: |
+        from lsst.ts.wep.task import EstimateZernikesDanishTask
+        config.estimateZernikes.retarget(EstimateZernikesDanishTask)
+      donutStampSelector.maxSelect: 5
+      donutStampSelector.maxFracBadPixels: 2.0e-4
+      estimateZernikes.binning: 2
+      estimateZernikes.nollIndices:
+        [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 20, 21, 22, 27, 28]
+
+# Define pipeline steps
+subsets:
+  step1:
+    subset:
+      - isr
+      - generateDonutDirectDetectTask
+      - cutOutDonutsScienceSensorGroupTask
+      - calcZernikesTask
+    description: |
+      This step processes the input images with ISR,
+      finds and cuts out the donut stamps,
+      and estimates the Zernike coefficients from the donut pairs.
+  step2a:
+    subset:
+      - aggregateZernikeTablesTask
+      - aggregateDonutTablesGroupTask
+      - aggregateAOSVisitTableTask
+      - plotAOSTask
+      - aggregateDonutStampsTask
+      - plotDonutTask
+      - plotPsfZernTask
+    description: |
+      AOS Donut visualization plotting tasks. This step generates plots
+      (including the pyramid residual and donut gallery) and
+      tables for the AOS visit.

pipelines/production/comCamRapidAnalysisPipeline.yaml

@@ -1,61 +1,23 @@
 description: rapid analysis pipeline for ComCam
 instrument: lsst.obs.lsst.LsstComCam
+imports:
+  - $TS_WEP_DIR/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml
+  - $TS_WEP_DIR/pipelines/_ingredients/donutVizGroupPipeline.yaml
+
 tasks:
-  generateDonutDirectDetectTask:
-    class: lsst.ts.wep.task.generateDonutDirectDetectTask.GenerateDonutDirectDetectTask
-    config:
-      donutSelector.useCustomMagLimit: True
-      donutSelector.sourceLimit: 20
-  cutOutDonutsScienceSensorGroupTask:
-    class: lsst.ts.wep.task.cutOutDonutsScienceSensorTask.CutOutDonutsScienceSensorTask
-    config:
-      python: |
-        from lsst.ts.wep.task.pairTask import GroupPairer
-        config.pairer.retarget(GroupPairer)
-      donutStampSize: 200
-      initialCutoutPadding: 40
   calcZernikesTask:
     class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask
     config:
-      estimateZernikes.nollIndices: [4, 5, 6, 7, 8, 9, 10, 11, 14, 15, 20, 21, 27, 28]
+      estimateZernikes.nollIndices: [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 18, 19, 20, 21, 22, 27, 28]
       estimateZernikes.convergeTol: 10.0e-9
+      estimateZernikes.compGain: 0.75
+      estimateZernikes.compSequence: [4, 4, 6, 6, 13, 13, 13, 13]
+      estimateZernikes.maxIter: 50
       estimateZernikes.requireConverge: True
       estimateZernikes.saveHistory: False
       estimateZernikes.maskKwargs: { "doMaskBlends": False }
       donutStampSelector.maxSelect: 5
-  isr:
-    class: lsst.ip.isr.IsrTaskLSST
-    config:
-      # Although we don't have to apply the amp offset corrections, we do want
-      # to compute them for analyzeAmpOffsetMetadata to report on as metrics.
-      doAmpOffset: true
-      ampOffset.doApplyAmpOffset: false
-      # Turn off slow steps in ISR
-      doBrighterFatter: false
-      # Mask saturated pixels,
-      # but turn off quadratic crosstalk because it's currently broken
-      doSaturation: True
-      crosstalk.doQuadraticCrosstalkCorrection: False
-  aggregateZernikeTablesTask:
-    class: lsst.donut.viz.AggregateZernikeTablesTask
-  aggregateDonutTablesGroupTask:
-      class: lsst.donut.viz.AggregateDonutTablesTask
-      config:
-          python: |
-              from lsst.ts.wep.task.pairTask import GroupPairer
-              config.pairer.retarget(GroupPairer)
-  aggregateAOSVisitTableTask:
-    class: lsst.donut.viz.AggregateAOSVisitTableTask
-  plotAOSTask:
-    class: lsst.donut.viz.PlotAOSTask
-    config:
-      doRubinTVUpload: true
-  aggregateDonutStampsTask:
-    class: lsst.donut.viz.AggregateDonutStampsTask
-  plotDonutTask:
-    class: lsst.donut.viz.PlotDonutTask
-    config:
-      doRubinTVUpload: true
+      donutStampSelector.maxFracBadPixels: 2.0e-4
 
 # Define pipeline steps
 subsets:
@@ -77,6 +39,7 @@ subsets:
       - plotAOSTask
       - aggregateDonutStampsTask
       - plotDonutTask
+      - plotPsfZernTask
     description: |
       AOS Donut visualization plotting tasks. This step generates plots
       (including the pyramid residual and donut gallery) and