Clean up and reformat USDF pipelines.

doc/versionHistory.rst

@@ -6,6 +6,14 @@
 Version History
 ##################
 
+.. _lsst.ts.wep-13.3.2:
+
+-------------
+13.3.2
+-------------
+
+* Clean up and reformat USDF pipelines.
+
 .. _lsst.ts.wep-13.3.1:
 
 -------------

pipelines/_ingredients/donutVizGroupPipeline.yaml

@@ -25,18 +25,3 @@ tasks:
     class: lsst.donut.viz.PlotPsfZernTask
     config:
       doRubinTVUpload: false
-
-subsets:
-  donutVizGroups:
-    subset:
-      - aggregateZernikeTablesTask
-      - aggregateDonutTablesGroupTask
-      - aggregateAOSVisitTableTask
-      - plotAOSTask
-      - aggregateDonutStampsTask
-      - plotDonutTask
-      - plotPsfZernTask
-    description: |
-      AOS Donut visualization plotting tasks. This step generates plots
-      (including the pyramid residual and donut gallery) and
-      tables for the AOS visit.

pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml

@@ -6,10 +6,10 @@ tasks:
     config:
       # Although we don't have to apply the amp offset corrections, we do want
       # to compute them for analyzeAmpOffsetMetadata to report on as metrics.
-      doAmpOffset: true
-      ampOffset.doApplyAmpOffset: false
+      doAmpOffset: True
+      ampOffset.doApplyAmpOffset: False
       # Turn off slow steps in ISR
-      doBrighterFatter: false
+      doBrighterFatter: False
       # Mask saturated pixels,
       # but turn off quadratic crosstalk because it's currently broken
       doSaturation: True
@@ -39,15 +39,3 @@ tasks:
       estimateZernikes.saveHistory: False
       estimateZernikes.maskKwargs: { "doMaskBlends": False }
       donutStampSelector.maxSelect: 5
-
-subsets:
-  wepDirectDetect:
-    subset:
-      - isr
-      - generateDonutDirectDetectTask
-      - cutOutDonutsScienceSensorGroupTask
-      - calcZernikesTask
-    description: |
-      This step processes the input images with ISR,
-      finds and cuts out the donut stamps,
-      and estimates the Zernike coefficients from the donut pairs.

pipelines/production/comCamBaselineUSDF_Danish.yaml

@@ -0,0 +1,56 @@
+description: Run WEP + DonutViz daily at USDF
+instrument: lsst.obs.lsst.LsstComCam
+imports:
+  - $TS_WEP_DIR/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml
+  - $TS_WEP_DIR/pipelines/_ingredients/donutVizGroupPipeline.yaml
+
+tasks:
+  generateDonutDirectDetectTask:
+    class: lsst.ts.wep.task.generateDonutDirectDetectTask.GenerateDonutDirectDetectTask
+    config:
+      donutSelector.useCustomMagLimit: True
+      donutSelector.sourceLimit: -1
+  calcZernikesTask:
+    class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask
+    config:
+      python: |
+        from lsst.ts.wep.task import EstimateZernikesDanishTask
+        config.estimateZernikes.retarget(EstimateZernikesDanishTask)
+      donutStampSelector.maxFracBadPixels: 2.0e-4
+      estimateZernikes.binning: 2
+      estimateZernikes.nollIndices:
+        [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 20, 21, 22, 27, 28]
+      estimateZernikes.saveHistory: False
+      estimateZernikes.lstsqKwargs:
+        ftol: 1.0e-3
+        xtol: 1.0e-3
+        gtol: 1.0e-3
+      donutStampSelector.maxSelect: -1
+
+# Define pipeline steps
+subsets:
+  step1a:
+    subset:
+      - isr
+      - generateDonutDirectDetectTask
+      - cutOutDonutsScienceSensorGroupTask
+    description: |
+      This step makes the input postISR images then
+      finds and cuts out the donut stamps.
+  step1b:
+    subset:
+      - calcZernikesTask
+    description: |
+      This step runs the Zernike calculation with danish.
+  step2:
+    subset:
+      - aggregateZernikeTablesTask
+      - aggregateDonutTablesGroupTask
+      - aggregateAOSVisitTableTask
+      - plotAOSTask
+      - aggregateDonutStampsTask
+      - plotDonutTask
+    description: |
+      AOS Donut visualization plotting tasks. This step generates plots
+      (including the pyramid residual and donut gallery) and
+      tables for the AOS visit.

pipelines/production/comCamDailyProcessing.yaml → pipelines/production/comCamBaselineUSDF_TIE.yaml

@@ -9,19 +9,37 @@ tasks:
     class: lsst.ts.wep.task.generateDonutDirectDetectTask.GenerateDonutDirectDetectTask
     config:
       donutSelector.useCustomMagLimit: True
+      donutSelector.sourceLimit: -1
+  calcZernikesTask:
+    class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask
+    config:
+      estimateZernikes.nollIndices: [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 18, 19, 20, 21, 22, 27, 28]
+      estimateZernikes.convergeTol: 10.0e-9
+      estimateZernikes.compGain: 0.75
+      estimateZernikes.compSequence: [4, 4, 6, 6, 13, 13, 13, 13]
+      estimateZernikes.maxIter: 50
+      estimateZernikes.requireConverge: True
+      estimateZernikes.saveHistory: False
+      estimateZernikes.maskKwargs: { "doMaskBlends": False }
+      donutStampSelector.maxFracBadPixels: 2.0e-4
+      donutStampSelector.maxSelect: -1
 
 # Define pipeline steps
 subsets:
   step1:
     subset:
+      - isr
       - generateDonutDirectDetectTask
       - cutOutDonutsScienceSensorGroupTask
+    description: |
+      This step makes the input postISR images then
+      finds and cuts out the donut stamps.
+  step1b:
+    subset:
       - calcZernikesTask
     description: |
-      This step processes the input images with ISR,
-      finds and cuts out the donut stamps,
-      and estimates the Zernike coefficients from the donut pairs.
-  step2a:
+      This step runs the Zernike calculation with danish.
+  step2:
     subset:
       - aggregateZernikeTablesTask
       - aggregateDonutTablesGroupTask

pipelines/production/comCamRefitWcsUSDF.yaml

@@ -0,0 +1,31 @@
+description: RefitWCS pipeline for ComCam
+instrument: lsst.obs.lsst.LsstComCam
+tasks:
+  generateDonutFromRefitWcsTask:
+    class: lsst.ts.wep.task.generateDonutFromRefitWcsTask.GenerateDonutFromRefitWcsTask
+    config:
+      connections.astromRefCat: "gaia_dr3_20230707"
+      connections.photoRefCat: "the_monster_20240904"
+      connections.exposure: "postISRCCD"
+      connections.outputExposure: "postFitPostISRCCD"
+      connections.fitDonutCatalog: "donutTable"
+      connections.donutCatalog: "refitWcsDonutTable"
+      donutSelector.useCustomMagLimit: True
+      astromRefFilter: "phot_g_mean"
+      photoRefFilterPrefix: "monster_SynthLSST"
+      astromTask.referenceSelector.doMagLimit: False
+      astromTask.maxIter: 3
+      astromTask.matcher.maxOffsetPix: 2000
+      astromTask.matcher.maxRotationDeg: 3.0
+      failTask: True
+      # donutSelector.unblendedSeparation: 1
+  cutOutDonutsScienceSensorGroupTask:
+    class: lsst.ts.wep.task.cutOutDonutsScienceSensorTask.CutOutDonutsScienceSensorTask
+    config:
+      connections.exposures: "postFitPostISRCCD"
+      connections.donutCatalog: "refitWcsDonutTable"
+      python: |
+        from lsst.ts.wep.task.pairTask import GroupPairer
+        config.pairer.retarget(GroupPairer)
+      donutStampSize: 200
+      initialCutoutPadding: 40

pipelines/production/comCamRefitWcsUSDF_Danish.yaml

@@ -0,0 +1,50 @@
+description: Run WEP + DonutViz daily at USDF
+instrument: lsst.obs.lsst.LsstComCam
+imports:
+  - $TS_WEP_DIR/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml
+  - $TS_WEP_DIR/pipelines/_ingredients/donutVizGroupPipeline.yaml
+
+tasks:
+  calcZernikesTask:
+    class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask
+    config:
+      python: |
+        from lsst.ts.wep.task import EstimateZernikesDanishTask
+        config.estimateZernikes.retarget(EstimateZernikesDanishTask)
+      donutStampSelector.maxFracBadPixels: 2.0e-4
+      estimateZernikes.binning: 2
+      estimateZernikes.nollIndices:
+        [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 20, 21, 22, 27, 28]
+      estimateZernikes.saveHistory: true
+      estimateZernikes.lstsqKwargs:
+        ftol: 1.0e-3
+        xtol: 1.0e-3
+        gtol: 1.0e-3
+      donutStampSelector.maxSelect: -1
+  aggregateDonutTablesGroupTask:
+    class: lsst.donut.viz.AggregateDonutTablesTask
+    config:
+        python: |
+            from lsst.ts.wep.task.pairTask import GroupPairer
+            config.pairer.retarget(GroupPairer)
+        connections.donutTables: "refitWcsDonutTable"
+
+# Define pipeline steps
+subsets:
+  step1b:
+    subset:
+      - calcZernikesTask
+    description: |
+      This step runs the Zernike calculation with danish.
+  step2:
+    subset:
+      - aggregateZernikeTablesTask
+      - aggregateDonutTablesGroupTask
+      - aggregateAOSVisitTableTask
+      - plotAOSTask
+      - aggregateDonutStampsTask
+      - plotDonutTask
+    description: |
+      AOS Donut visualization plotting tasks. This step generates plots
+      (including the pyramid residual and donut gallery) and
+      tables for the AOS visit.

pipelines/production/comCamRefitWcsUSDF_TIE.yaml

@@ -0,0 +1,47 @@
+description: Run WEP + DonutViz daily at USDF
+instrument: lsst.obs.lsst.LsstComCam
+imports:
+  - $TS_WEP_DIR/pipelines/_ingredients/wepDirectDetectScienceGroupPipeline.yaml
+  - $TS_WEP_DIR/pipelines/_ingredients/donutVizGroupPipeline.yaml
+
+tasks:
+  calcZernikesTask:
+    class: lsst.ts.wep.task.calcZernikesTask.CalcZernikesTask
+    config:
+      estimateZernikes.nollIndices: [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 18, 19, 20, 21, 22, 27, 28]
+      estimateZernikes.convergeTol: 10.0e-9
+      estimateZernikes.compGain: 0.75
+      estimateZernikes.compSequence: [4, 4, 6, 6, 13, 13, 13, 13]
+      estimateZernikes.maxIter: 50
+      estimateZernikes.requireConverge: True
+      estimateZernikes.saveHistory: True
+      estimateZernikes.maskKwargs: { "doMaskBlends": False }
+      donutStampSelector.maxFracBadPixels: 2.0e-4
+      donutStampSelector.maxSelect: -1
+  aggregateDonutTablesGroupTask:
+    class: lsst.donut.viz.AggregateDonutTablesTask
+    config:
+        python: |
+            from lsst.ts.wep.task.pairTask import GroupPairer
+            config.pairer.retarget(GroupPairer)
+        connections.donutTables: "refitWcsDonutTable"
+
+# Define pipeline steps
+subsets:
+  step1b:
+    subset:
+      - calcZernikesTask
+    description: |
+      This step runs the Zernike calculation with danish.
+  step2:
+    subset:
+      - aggregateZernikeTablesTask
+      - aggregateDonutTablesGroupTask
+      - aggregateAOSVisitTableTask
+      - plotAOSTask
+      - aggregateDonutStampsTask
+      - plotDonutTask
+    description: |
+      AOS Donut visualization plotting tasks. This step generates plots
+      (including the pyramid residual and donut gallery) and
+      tables for the AOS visit.

python/lsst/ts/wep/task/cutOutDonutsBase.py

@@ -695,9 +695,10 @@ def cutOutStamps(self, exposure, donutCatalog, defocalType, cameraName):
         stampsMetadata = PropertyList()
         stampsMetadata["VISIT"] = np.array([visitId] * catalogLength, dtype=int)
         # Save the donut flux as magnitude
-        if len(donutCatalog["source_flux"]) > 0:
+        fluxLabel = next((colName for colName in donutCatalog.columns if colName.endswith(f'{bandLabel}_flux')), None)
+        if fluxLabel is not None and len(donutCatalog[fluxLabel]) > 0:
             stampsMetadata["MAG"] = (
-                donutCatalog["source_flux"].value * u.nJy
+                donutCatalog[fluxLabel].value * u.nJy
             ).to_value(u.ABmag)
         else:
             stampsMetadata["MAG"] = np.array([])

python/lsst/ts/wep/task/generateDonutCatalogUtils.py

@@ -85,7 +85,11 @@ def runSelection(refObjLoader, detector, wcs, filterName, donutSelectorTask):
 
 
 def donutCatalogToAstropy(
-    donutCatalog=None, filterName=None, blendCentersX=None, blendCentersY=None
+    donutCatalog=None,
+    filterName=None,
+    blendCentersX=None,
+    blendCentersY=None,
+    sortFilterIdx=0,
 ):
     """
     Reformat afwCatalog into an astropy QTable sorted by flux with
@@ -98,19 +102,22 @@ def donutCatalogToAstropy(
         ReferenceObjectLoader search over the detector footprint.
         If None then it will return an empty QTable.
         (the default is None.)
-    filterName : `str` or `None`, optional
-        Name of camera filter. If donutCatalog is not None then
+    filterName : `list` or `None`, optional
+        Name of catalog flux filters. If donutCatalog is not None then
         this cannot be None. (the default is None.)
     blendCentersX : `list` or `None`, optional
-            X pixel position of centroids for blended objects. List
-            should be the same length as the donutCatalog. If
-            blendCentersY is not None then this cannot be None. (the default
-            is None.)
+        X pixel position of centroids for blended objects. List
+        should be the same length as the donutCatalog. If
+        blendCentersY is not None then this cannot be None. (the default
+        is None.)
     blendCentersY : `list` or `None`, optional
-            Y pixel position of centroids for blended objects. List
-            should be the same length as the donutCatalog. If
-            blendCentersX is not None then this cannot be None. (the default
-            is None.)
+        Y pixel position of centroids for blended objects. List
+        should be the same length as the donutCatalog. If
+        blendCentersX is not None then this cannot be None. (the default
+        is None.)
+    sortFilterIdx : int, optional
+        Index for which filter in filterName to sort the entire catalog
+        by brightness. (the default is 0.)
 
     Returns
     -------
@@ -144,12 +151,12 @@ def donutCatalogToAstropy(
         dec = donutCatalog["coord_dec"]
         centroidX = donutCatalog["centroid_x"]
         centroidY = donutCatalog["centroid_y"]
-        sourceFlux = donutCatalog[f"{filterName}_flux"]
+        sourceFlux = [donutCatalog[f"{fName}_flux"] for fName in filterName]
 
         if (blendCentersX is None) and (blendCentersY is None):
             blendCX = list()
             blendCY = list()
-            for idx in range(len(donutCatalog)):
+            for _ in range(len(donutCatalog)):
                 blendCX.append(list())
                 blendCY.append(list())
         elif isinstance(blendCentersX, list) and isinstance(blendCentersY, list):
@@ -178,16 +185,17 @@ def donutCatalogToAstropy(
             )
             raise ValueError(blendErrMsg)
 
-    flux_sort = np.argsort(sourceFlux)[::-1]
+    flux_sort = np.argsort(sourceFlux[sortFilterIdx])[::-1]
 
     fieldObjects = QTable()
     fieldObjects["coord_ra"] = ra * u.rad
     fieldObjects["coord_dec"] = dec * u.rad
     fieldObjects["centroid_x"] = centroidX
     fieldObjects["centroid_y"] = centroidY
-    fieldObjects["source_flux"] = sourceFlux * u.nJy
+    for idx in range(len(filterName)):
+        fieldObjects[f"{filterName[idx]}_flux"] = sourceFlux[idx] * u.nJy
 
-    fieldObjects.sort("source_flux", reverse=True)
+    fieldObjects.sort(f"{filterName[sortFilterIdx]}_flux", reverse=True)
 
     fieldObjects.meta["blend_centroid_x"] = [blendCX[idx] for idx in flux_sort]
     fieldObjects.meta["blend_centroid_y"] = [blendCY[idx] for idx in flux_sort]