From 7ffed92efc59ebfcd6cd18e9dfb4328f0cae99c8 Mon Sep 17 00:00:00 2001 From: dmggh Date: Wed, 23 Aug 2023 22:42:12 -0400 Subject: [PATCH] (JP-3337) Renaming of undersampling_correction to charge_migration. (#7825) Co-authored-by: Howard Bushouse --- CHANGES.rst | 9 +- docs/jwst/charge_migration/arguments.rst | 7 + docs/jwst/charge_migration/description.rst | 51 ++++ docs/jwst/charge_migration/index.rst | 14 ++ .../jwst/charge_migration/reference_files.rst | 3 + docs/jwst/package_index.rst | 2 +- docs/jwst/pipeline/calwebb_detector1.rst | 6 +- docs/jwst/ramp_fitting/description.rst | 4 +- .../references_general/references_general.rst | 2 +- jwst/charge_migration/__init__.py | 3 + jwst/charge_migration/charge_migration.py | 143 ++++++++++++ .../charge_migration/charge_migration_step.py | 46 ++++ .../tests/test_charge_migration.py | 219 ++++++++++++++++++ jwst/lib/suffix.py | 2 +- jwst/pipeline/calwebb_detector1.py | 8 +- jwst/ramp_fitting/ramp_fit_step.py | 14 +- jwst/ramp_fitting/tests/test_ramp_fit.py | 12 +- jwst/regtest/test_niriss_image.py | 8 +- jwst/step.py | 5 +- jwst/stpipe/integration.py | 2 +- 20 files changed, 527 insertions(+), 33 deletions(-) create mode 100644 docs/jwst/charge_migration/arguments.rst create mode 100644 docs/jwst/charge_migration/description.rst create mode 100644 docs/jwst/charge_migration/index.rst create mode 100644 docs/jwst/charge_migration/reference_files.rst create mode 100644 jwst/charge_migration/__init__.py create mode 100644 jwst/charge_migration/charge_migration.py create mode 100755 jwst/charge_migration/charge_migration_step.py create mode 100644 jwst/charge_migration/tests/test_charge_migration.py diff --git a/CHANGES.rst b/CHANGES.rst index b44e73729e9..aba39c7a45e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -28,6 +28,13 @@ calwebb_spec2 that is returned by the pipeline to ensure a file is created with the expected ``_cal`` suffix. [#7772] +charge_migration +---------------- + +- Step was renamed from undersampling_migration. Changed default signal threshold, + added efficient routine to flag neighborhood pixels, added new unit test, + improved earlier unit tests, updated docs. [#7825] + cube_build ---------- @@ -110,7 +117,7 @@ residual_fringe - Use scipy.interpolate.BSpline instead of astropy.modeling.Spline1D in residual_fringe fitting utils [#7764] - + undersampling_correction ------------------------ diff --git a/docs/jwst/charge_migration/arguments.rst b/docs/jwst/charge_migration/arguments.rst new file mode 100644 index 00000000000..8a4e8d02e2e --- /dev/null +++ b/docs/jwst/charge_migration/arguments.rst @@ -0,0 +1,7 @@ +Arguments +========= + +The ``charge migration`` step has one optional argument that can be set by the user: + +* ``--signal_threshold``: A floating-point value in units of ADU for the science value above which + a group's DQ will be flagged as CHARGELOSS and DO_NOT_USE. diff --git a/docs/jwst/charge_migration/description.rst b/docs/jwst/charge_migration/description.rst new file mode 100644 index 00000000000..e893e8d8825 --- /dev/null +++ b/docs/jwst/charge_migration/description.rst @@ -0,0 +1,51 @@ +Description +=========== + +:Class: `jwst.charge_migration.ChargeMigrationStep` +:Alias: charge_migration + + +Overview +-------- +This step corrects for an artifact seen in undersampled NIRISS images that may depress flux +in resampled images. The artifact is seen in dithered images where the star is centered in +a pixel. When the peak pixels of such stars approach the saturation level, they suffer from +significant :ref:`charge migration `: +the spilling of charge from a saturated pixel into its neighboring pixels. This charge migration +causes group-to-group differences to decrease significantly once the signal level is greater than +~25,000 ADU. As a result, the last several groups of these ramps get flagged by the :ref:`jump ` +step. The smaller number of groups used for these pixels in the :ref:`ramp_fitting ` +step results in them having larger read noise variances, which in turn leads to lower weights used +during resampling. This ultimately leads to a lower than normal flux for the star in resampled images. + +Once a group in a ramp has been flagged as affected by charge migration, all subsequent +groups in the ramp are also flagged. By flagging these groups, they are not used in the +computation of slopes in the :ref:`ramp_fitting ` step. However, as described +in the algorithm section below, they _are_ used in the calculation of the variance of the slope +due to readnoise. + +Input details +------------- +The input must be in the form of a `~jwst.datamodels.RampModel`. + + +Algorithm +--------- +The first group, and all subsequent groups, exceeding the value of the +``signal_threshold`` parameter is flagged as CHARGELOSS. ``signal_threshold`` is in units +of ADUs. These groups will also be flagged as DO_NOT_USE, and will not +be included in the slope calculation during the ``ramp_fitting`` step. Despite being flagged +as DO_NOT_USE, these CHARGELOSS groups are still included in the calculation of the +variance due to readnoise. +This results in a readnoise variance for undersampled pixels that is similar to that of +pixels unaffected by charge migration. For the Poisson noise variance calculation in +:ref:`ramp_fitting `, the CHARGELOSS/DO_NOT_USE groups are not included. + +For integrations having only 1 or 2 groups, no flagging will be performed. + + +Output product +-------------- +The output is a new copy of the input `~jwst.datamodels.RampModel`, with the updated DQ flags +added to the GROUPDQ array. + diff --git a/docs/jwst/charge_migration/index.rst b/docs/jwst/charge_migration/index.rst new file mode 100644 index 00000000000..6ae2e999065 --- /dev/null +++ b/docs/jwst/charge_migration/index.rst @@ -0,0 +1,14 @@ +.. _charge_migration_step: + +================ +Charge Migration +================ + +.. toctree:: + :maxdepth: 2 + + description.rst + arguments.rst + reference_files.rst + +.. automodapi:: jwst.charge_migration diff --git a/docs/jwst/charge_migration/reference_files.rst b/docs/jwst/charge_migration/reference_files.rst new file mode 100644 index 00000000000..d9c23224fa2 --- /dev/null +++ b/docs/jwst/charge_migration/reference_files.rst @@ -0,0 +1,3 @@ +Reference Files +=============== +This step does not use any reference files. diff --git a/docs/jwst/package_index.rst b/docs/jwst/package_index.rst index ab8c05cca94..19180c55523 100644 --- a/docs/jwst/package_index.rst +++ b/docs/jwst/package_index.rst @@ -14,6 +14,7 @@ Package Index background_step/index.rst background_subtraction/index.rst barshadow/index.rst + charge_migration/index.rst combine_1d/index.rst cube_build/index.rst dark_current/index.rst @@ -63,7 +64,6 @@ Package Index superbias/index.rst tso_photometry/index.rst tweakreg/index.rst - undersampling_correction/index.rst wavecorr/index.rst wfs_combine/index.rst wfss_contam/index.rst diff --git a/docs/jwst/pipeline/calwebb_detector1.rst b/docs/jwst/pipeline/calwebb_detector1.rst index 07e78aa4471..0060592df1f 100644 --- a/docs/jwst/pipeline/calwebb_detector1.rst +++ b/docs/jwst/pipeline/calwebb_detector1.rst @@ -72,8 +72,8 @@ on either the first group or frame zero pixel values. +----------------------------------------------------------------------+---------+---------+-----------------------------------------+---------+---------+ | | | | :ref:`refpix ` | |check| | |check| | +----------------------------------------------------------------------+---------+---------+-----------------------------------------+---------+---------+ -| :ref:`undersampling_correction ` [3]_ | |check| | | | | | -+----------------------------------------------------------------------+---------+---------+----------------------+------------------+---------+---------+ +| :ref:`charge_migration ` [3]_ | |check| | | | | | ++----------------------------------------------------------------------+---------+---------+-----------------------------------------+---------+---------+ | :ref:`jump ` | |check| | |check| | :ref:`jump ` | |check| | |check| | +----------------------------------------------------------------------+---------+---------+-----------------------------------------+---------+---------+ | :ref:`ramp_fitting ` | |check| | |check| | :ref:`ramp_fitting ` | |check| | |check| | @@ -86,7 +86,7 @@ on either the first group or frame zero pixel values. retrieved from CRDS will skip the :ref:`ipc ` step for all instruments. .. [2] The :ref:`persistence ` step is currently hardwired to be skipped in the `Detector1Pipeline` module for all NIRSpec exposures. -.. [3] By default, the :ref:`undersampling_correction ` step is skipped in +.. [3] By default, the :ref:`charge_migration ` step is skipped in the `Detector1Pipeline` module for all instruments. diff --git a/docs/jwst/ramp_fitting/description.rst b/docs/jwst/ramp_fitting/description.rst index f6df8473de6..5174cbe4a2a 100644 --- a/docs/jwst/ramp_fitting/description.rst +++ b/docs/jwst/ramp_fitting/description.rst @@ -322,9 +322,9 @@ extension. Weighted Readnoise Variance +++++++++++++++++++++++++++ -If the :ref:`undersampling correction ` +If the :ref:`charge migration ` step has been performed prior to ramp fitting, any group whose value exceeds the -`signal_threshold` parameter will have been flagged with the UNDERSAMP and DO_NOT_USE +`signal_threshold` parameter will have been flagged with the CHARGELOSS and DO_NOT_USE data quality flags. Due to the DO_NOT_USE flags, such groups will be excluded from the slope calculations. diff --git a/docs/jwst/references_general/references_general.rst b/docs/jwst/references_general/references_general.rst index cf7d4ba3967..c10a039a4fa 100644 --- a/docs/jwst/references_general/references_general.rst +++ b/docs/jwst/references_general/references_general.rst @@ -602,7 +602,7 @@ Bit Value Name Description 4 16 OUTLIER Flagged by outlier detection 5 32 PERSISTENCE High persistence 6 64 AD_FLOOR Below A/D floor -7 128 UNDERSAMP Undersampling correction +7 128 CHARGELOSS Charge Migration 8 256 UNRELIABLE_ERROR Uncertainty exceeds quoted error 9 512 NON_SCIENCE Pixel not on science portion of detector 10 1024 DEAD Dead pixel diff --git a/jwst/charge_migration/__init__.py b/jwst/charge_migration/__init__.py new file mode 100644 index 00000000000..07bec3f47a9 --- /dev/null +++ b/jwst/charge_migration/__init__.py @@ -0,0 +1,3 @@ +from .charge_migration_step import ChargeMigrationStep + +__all__ = ['ChargeMigrationStep'] diff --git a/jwst/charge_migration/charge_migration.py b/jwst/charge_migration/charge_migration.py new file mode 100644 index 00000000000..30e1b9bc1c2 --- /dev/null +++ b/jwst/charge_migration/charge_migration.py @@ -0,0 +1,143 @@ +# Module for charge migration +# +import logging +import numpy as np + +from stdatamodels.jwst.datamodels import dqflags + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + +GOOD = dqflags.group["GOOD"] +DNU = dqflags.group["DO_NOT_USE"] +CHLO = dqflags.group["CHARGELOSS"] + +CHLO_DNU = CHLO + DNU + + +def charge_migration(input_model, signal_threshold): + """ + Correct for chargemigration + + Parameters + ---------- + input_model : `~jwst.datamodels.RampModel` + The input science data to be corrected + + signal_threshold : float + Science value above which a group will be flagged as CHARGELOSS + + Returns + ------- + output_model : `~jwst.datamodels.RampModel` + Data model with charge_migration applied; add CHARGELOSS and + + DO_NOT_USE flags to groups exceeding signal_threshold + """ + data = input_model.data + gdq = input_model.groupdq + + # Create the output model as a copy of the input + output_model = input_model.copy() + + log.info('Using signal_threshold: %.2f', signal_threshold) + + gdq_new = flag_pixels(data, gdq, signal_threshold) + + # Save the flags in the output GROUPDQ array + output_model.groupdq = gdq_new + + return output_model + + +def flag_pixels(data, gdq, signal_threshold): + """ + Flag each group in each ramp that exceeds signal_threshold as CHARGELOSS and DO_NOT_USE, + skipping groups already flagged as DO_NOT_USE. + + Parameters + ---------- + data : float, 4D array + science array + + gdq : int, 4D array + group dq array + + signal_threshold : float + Science value above which a group will be flagged as CHARGELOSS and DO_NOT_USE + + Returns + ------- + new_gdq : int, 4D array + updated group dq array + """ + n_ints, n_grps, n_rows, n_cols = gdq.shape + + new_gdq = gdq.copy() # Updated gdq + + # Flag all exceedances with CHARGELOSS and NO_NOT_USE + chargeloss_pix = (data > signal_threshold) & (gdq != DNU) + new_gdq[chargeloss_pix] = np.bitwise_or(new_gdq[chargeloss_pix], CHLO | DNU) + + # Reset groups previously flagged as DNU + gdq_orig = gdq.copy() # For resetting to previously flagged DNU + wh_gdq_DNU = np.bitwise_and(gdq_orig, DNU) + + # Get indices for exceedances + arg_where = np.argwhere(new_gdq == CHLO_DNU) + + a_int = arg_where[:, 0] # array of integrations + a_grp = arg_where[:, 1] # array of groups + a_row = arg_where[:, 2] # array of rows + a_col = arg_where[:, 3] # array of columns + + # Process the 4 nearest neighbors of each exceedance + # Pixel to the east + xx_max_p1 = a_col[a_col < (n_cols-1)] + 1 + i_int = a_int[a_col < (n_cols-1)] + i_grp = a_grp[a_col < (n_cols-1)] + i_row = a_row[a_col < (n_cols-1)] + + if len(xx_max_p1) > 0: + new_gdq[i_int, i_grp, i_row, xx_max_p1] = \ + np.bitwise_or(new_gdq[i_int, i_grp, i_row, xx_max_p1], CHLO | DNU) + + new_gdq[wh_gdq_DNU == 1] = gdq_orig[wh_gdq_DNU == 1] # reset for earlier DNUs + + # Pixel to the west + xx_m1 = a_col[a_col > 0] - 1 + i_int = a_int[a_col > 0] + i_grp = a_grp[a_col > 0] + i_row = a_row[a_col > 0] + + if len(xx_m1) > 0: + new_gdq[i_int, i_grp, i_row, xx_m1] = \ + np.bitwise_or(new_gdq[i_int, i_grp, i_row, xx_m1], CHLO | DNU) + + new_gdq[wh_gdq_DNU == 1] = gdq_orig[wh_gdq_DNU == 1] # reset for earlier DNUs + + # Pixel to the north + yy_m1 = a_row[a_row > 0] - 1 + i_int = a_int[a_row > 0] + i_grp = a_grp[a_row > 0] + i_col = a_col[a_row > 0] + + if len(yy_m1) > 0: + new_gdq[i_int, i_grp, yy_m1, i_col] = \ + np.bitwise_or(new_gdq[i_int, i_grp, yy_m1, i_col], CHLO | DNU) + + new_gdq[wh_gdq_DNU == 1] = gdq_orig[wh_gdq_DNU == 1] # reset for earlier DNUs + + # Pixel to the south + yy_max_p1 = a_row[a_row < (n_rows-1)] + 1 + i_int = a_int[a_row < (n_rows-1)] + i_grp = a_grp[a_row < (n_rows-1)] + i_col = a_col[a_row < (n_rows-1)] + + if len(yy_max_p1) > 0: + new_gdq[i_int, i_grp, yy_max_p1, i_col] = \ + np.bitwise_or(new_gdq[i_int, i_grp, yy_max_p1, i_col], CHLO | DNU) + + new_gdq[wh_gdq_DNU == 1] = gdq_orig[wh_gdq_DNU == 1] # reset for earlier DNUs + + return new_gdq diff --git a/jwst/charge_migration/charge_migration_step.py b/jwst/charge_migration/charge_migration_step.py new file mode 100755 index 00000000000..f2e03ca1ef0 --- /dev/null +++ b/jwst/charge_migration/charge_migration_step.py @@ -0,0 +1,46 @@ +#! /usr/bin/env python +import logging + +from ..stpipe import Step + +from stdatamodels.jwst import datamodels + +from . import charge_migration + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + +__all__ = ["ChargeMigrationStep"] + + +class ChargeMigrationStep(Step): + """ + This Step sets the CHARGELOSS flag for groups exhibiting significant + charge migration. + """ + class_alias = "charge_migration" + + spec = """ + signal_threshold = float(default=30000) + skip = boolean(default=True) + """ + + def process(self, input): + + # Open the input data model + with datamodels.RampModel(input) as input_model: + if (input_model.data.shape[1] < 3): # skip step if only 1 or 2 groups/integration + log.info('Too few groups per integration; skipping charge_migration') + + result = input_model + result.meta.cal_step.charge_migration = 'SKIPPED' + + return result + + # Retrieve the parameter value(s) + signal_threshold = self.signal_threshold + + result = charge_migration.charge_migration(input_model, signal_threshold) + result.meta.cal_step.charge_migration = 'COMPLETE' + + return result diff --git a/jwst/charge_migration/tests/test_charge_migration.py b/jwst/charge_migration/tests/test_charge_migration.py new file mode 100644 index 00000000000..ce307b12ad8 --- /dev/null +++ b/jwst/charge_migration/tests/test_charge_migration.py @@ -0,0 +1,219 @@ +import numpy as np + +from stdatamodels.jwst.datamodels import RampModel +from stdatamodels.jwst.datamodels import dqflags + +from jwst.charge_migration.charge_migration import charge_migration +from jwst.charge_migration.charge_migration_step import ChargeMigrationStep + +import numpy.testing as npt + +test_dq_flags = dqflags.pixel +GOOD = test_dq_flags["GOOD"] +DNU = test_dq_flags["DO_NOT_USE"] +CHLO = test_dq_flags["CHARGELOSS"] + +DROU = test_dq_flags["DROPOUT"] +CHLO_DNU = CHLO + DNU + + +def test_pix_0(): + """ + Having all data in ramp below the signal threshold, the only non-GOOD + groups in the output GROUPDQ should be those DNU propagated from the input. + """ + ngroups, nints, nrows, ncols = 10, 1, 1, 1 + ramp_model, pixdq, groupdq, err = create_mod_arrays( + ngroups, nints, nrows, ncols) + + signal_threshold = 30000. + + # Populate pixel-specific SCI and GROUPDQ arrays + # Set all SCI to be below the signal threshold, and some input + # GROUPDQ to be DNU + ramp_model.data[0, :, 0, 0] = np.array((signal_threshold - 100.), dtype=np.float32) + ramp_model.groupdq[0, :, 0, 0] = [GOOD, DNU, GOOD, DNU, DNU, GOOD, GOOD, GOOD, GOOD, DNU] + in_gdq = ramp_model.groupdq + + out_model = charge_migration(ramp_model, signal_threshold) + out_gdq = out_model.groupdq + + npt.assert_array_equal(out_gdq, in_gdq) + + +def test_pix_1(): + """ + Tests groups whose data exceeds the signal threshold; 1 group is already + flagged as DNU from a previous calibration step, and 1 group is GOOD. + Also tests groups whose data does not exceed the signal threshold; + similarly 1 group is already flagged as DNU from a previous calibration + step, and 1 is GOOD. All data beyond the first exceedance are also + flagged as CHARGELOSS and DNU. + """ + ngroups, nints, nrows, ncols = 10, 1, 1, 1 + ramp_model, pixdq, groupdq, err = create_mod_arrays( + ngroups, nints, nrows, ncols) + + signal_threshold = 20000. + + # Populate SCI and GROUPDQ arrays. + ramp_model.data[0, 1, 0, 0] = np.array((0.5 * signal_threshold), dtype=np.float32) + + ramp_model.data[0, 2, 0, 0] = np.array((0.8 * signal_threshold), dtype=np.float32) + ramp_model.groupdq[0, 2, 0, 0] = DNU # should not get CHLO, not an exceedance + + ramp_model.data[0, 3, 0, 0] = np.array((signal_threshold + 5000.), dtype=np.float32) + ramp_model.groupdq[0, 3, 0, 0] = DNU # should not get CHLO, although exceedance + + ramp_model.data[0, 4:, 0, 0] = np.array((signal_threshold + 6000.), dtype=np.float32) + ramp_model.groupdq[0, 4:, 0, 0] = GOOD + + true_out_gdq = ramp_model.groupdq.copy() + true_out_gdq[0, 2, 0, 0] = DNU + true_out_gdq[0, 3, 0, 0] = DNU + true_out_gdq[0, 4:, 0, 0] = CHLO_DNU + + out_model = charge_migration(ramp_model, signal_threshold) + out_gdq = out_model.groupdq + + npt.assert_array_equal(out_gdq, true_out_gdq) + + +def test_too_few_groups(): + """ + Test that processing for datasets having too few (<3) groups per integration + are skipped. + """ + ngroups, nints, nrows, ncols = 2, 1, 1, 1 + ramp_model, pixdq, groupdq, err = create_mod_arrays( + ngroups, nints, nrows, ncols) + + ramp_model.data[0, :, 0, 0] = 20000. + sig_thresh = 100. + + result = ChargeMigrationStep.call(ramp_model, skip=False, + signal_threshold=sig_thresh) + status = result.meta.cal_step.charge_migration + + npt.assert_string_equal(status, "SKIPPED") + + +def test_flag_neighbors(): + """ + Test flagging of 4 nearest neighbors of exceedances. Tests pixels on + array edges, Tests exclusion of groups previously flagged as DO_NOT_USE. + """ + ngroups, nints, nrows, ncols = 6, 1, 4, 3 + ramp_model, pixdq, groupdq, err = create_mod_arrays( + ngroups, nints, nrows, ncols) + + signal_threshold = 4400. + + # Populate pixel-specific SCI and GROUPDQ arrays + ramp_model.data[0, :, :, :] = \ + np.array([[ + [1900., 2666., 2100.], + [3865., 2300., 3177.], + [3832., 3044., 3588.], + [3799., 3233., 3000.]], + + [[2100., 2866., 2300.], + [4065., 2500., 3377.], + [4032., 3244., 3788.], + [3999., 3433., 3200.]], + + [[2300., 3066., 2500.], + [4265., 2700., 3577.], + [4232., 3444., 3988.], + [4199., 3633., 3400.]], + + [[2500., 3266., 2700.], + [4465., 2900., 3777.], + [4432., 3644., 4188.], + [4399., 3833., 3600.]], + + [[2700., 3466., 2900.], + [4665., 3100., 3977.], + [4632., 3844., 4388.], + [4599., 4033., 3800.]], + + [[2900., 3666., 3100.], + [4865., 3300., 4177.], + [4832., 4044., 4588.], + [4799., 4233., 4000.]]], dtype=np.float32) + + # These group DQ values should propagate unchanged to the output + ramp_model.groupdq[:, 4, 2, 0] = [DNU] + ramp_model.groupdq[:, 1, 2, 2] = [DNU] + ramp_model.groupdq[:, 2, 1, 1] = [DROU + DNU] + + out_model = charge_migration(ramp_model, signal_threshold) + + out_gdq = out_model.groupdq + + true_out_gdq = ramp_model.groupdq.copy() + true_out_gdq[0, :, :, :] = \ + np.array([[ + [0, 0, 0], + [0, 0, 0], + [0, 0, 0], + [0, 0, 0]], + + [[0, 0, 0], + [0, 0, 0], + [0, 0, DNU], + [0, 0, 0]], + + [[0, 0, 0], + [0, 9, 0], + [0, 0, 0], + [0, 0, 0]], + + [[CHLO_DNU, 0, 0], + [CHLO_DNU, CHLO_DNU, 0], + [CHLO_DNU, CHLO_DNU, 0], + [CHLO_DNU, 0, 0]], + + [[CHLO_DNU, 0, 0], + [CHLO_DNU, CHLO_DNU, 0], + [DNU, 0, 0], + [CHLO_DNU, CHLO_DNU, 0]], + + [[CHLO_DNU, 0, 0], + [CHLO_DNU, CHLO_DNU, CHLO_DNU], + [CHLO_DNU, CHLO_DNU, CHLO_DNU], + [CHLO_DNU, CHLO_DNU, CHLO_DNU]]], dtype=np.uint8) + + npt.assert_array_equal(out_gdq, true_out_gdq) + + +def create_mod_arrays(ngroups, nints, nrows, ncols): + """ + For an input datacube (NIRISS), create arrays having + the specified dimensions for the pixel DQ, the group DQ, and the + ERR extensions, and create datamodels for the ramp. All groups + in all arrays have initial values set to 0. + """ + err = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.float32) + data = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.float32) + pixdq = np.zeros(shape=(nrows, ncols), dtype=np.uint32) + gdq = np.zeros(shape=(nints, ngroups, nrows, ncols), dtype=np.uint8) + + # Create and populate ramp model + ramp_model = RampModel(data=data, err=err, pixeldq=pixdq, groupdq=gdq) + ramp_model.meta.instrument.name = 'NIRISS' + ramp_model.meta.instrument.detector = 'NIS' + + ramp_model.meta.subarray.name = 'FULL' + ramp_model.meta.subarray.xstart = 1 + ramp_model.meta.subarray.ystart = 1 + ramp_model.meta.subarray.xsize = ncols + ramp_model.meta.subarray.ysize = nrows + + ramp_model.meta.exposure.ngroups = ngroups + ramp_model.meta.exposure.nframes = 1 + ramp_model.meta.exposure.groupgap = 0 + ramp_model.meta.exposure.drop_frames1 = 0 + ramp_model.meta.exposure.type = 'NIS_IMAGE' + + return ramp_model, pixdq, gdq, err diff --git a/jwst/lib/suffix.py b/jwst/lib/suffix.py index 36b62b30d91..8c934432239 100644 --- a/jwst/lib/suffix.py +++ b/jwst/lib/suffix.py @@ -189,7 +189,7 @@ 'assign_mtwcs', 'wavecorrstep', 'wfsscontamstep', - 'undersamplingcorrectionstep', + 'chargemigrationstep', 'pixelreplacestep', } diff --git a/jwst/pipeline/calwebb_detector1.py b/jwst/pipeline/calwebb_detector1.py index 889e5f492f6..ed85ba213cd 100644 --- a/jwst/pipeline/calwebb_detector1.py +++ b/jwst/pipeline/calwebb_detector1.py @@ -19,7 +19,7 @@ from ..dark_current import dark_current_step from ..reset import reset_step from ..persistence import persistence_step -from ..undersampling_correction import undersampling_correction_step +from ..charge_migration import charge_migration_step from ..jump import jump_step from ..ramp_fitting import ramp_fit_step from ..gain_scale import gain_scale_step @@ -60,7 +60,7 @@ class Detector1Pipeline(Pipeline): 'dark_current': dark_current_step.DarkCurrentStep, 'reset': reset_step.ResetStep, 'persistence': persistence_step.PersistenceStep, - 'undersampling_correction': undersampling_correction_step.UndersamplingCorrectionStep, + 'charge_migration': charge_migration_step.ChargeMigrationStep, 'jump': jump_step.JumpStep, 'ramp_fit': ramp_fit_step.RampFitStep, 'gain_scale': gain_scale_step.GainScaleStep, @@ -119,8 +119,8 @@ def process(self, input): input = self.dark_current(input) - # apply the undersampling_correction step - input = self.undersampling_correction(input) + # apply the charge_migration step + input = self.charge_migration(input) # apply the jump step input = self.jump(input) diff --git a/jwst/ramp_fitting/ramp_fit_step.py b/jwst/ramp_fitting/ramp_fit_step.py index df1bc61ab27..b88934a82ae 100644 --- a/jwst/ramp_fitting/ramp_fit_step.py +++ b/jwst/ramp_fitting/ramp_fit_step.py @@ -187,7 +187,7 @@ def compute_RN_variances(groupdq, readnoise_2d, gain_2d, group_time): ---------- groupdq : ndarray The group data quality array for the exposure, 4-D flag. - For groups that have been flagged as both UNDERSAMP and + For groups that have been flagged as both CHARGELOSS and DO_NOT_USE, both flags have been reset. readnoise_2d : ndarray @@ -451,15 +451,15 @@ def process(self, input): self.algorithm, self.weighting, max_cores, dqflags.pixel, suppress_one_group=self.suppress_one_group) - # Create a gdq to modify if there are undersampling-corrected groups + # Create a gdq to modify if there are charge_migrated groups gdq = input_model_W.groupdq.copy() - # Locate groups where that are flagged with UNDERSAMP - wh_undersamp = np.where(np.bitwise_and(gdq.astype(np.uint32), dqflags.group['UNDERSAMP'])) + # Locate groups where that are flagged with CHARGELOSS + wh_chargeloss = np.where(np.bitwise_and(gdq.astype(np.uint32), dqflags.group['CHARGELOSS'])) - if len(wh_undersamp[0]) > 0: - # Unflag groups flagged as both UNDERSAMP and DO_NOT_USE - gdq[wh_undersamp] -= (dqflags.group['DO_NOT_USE'] + dqflags.group['UNDERSAMP']) + if len(wh_chargeloss[0]) > 0: + # Unflag groups flagged as both CHARGELOSS and DO_NOT_USE + gdq[wh_chargeloss] -= (dqflags.group['DO_NOT_USE'] + dqflags.group['CHARGE_LOSS']) # Flag SATURATED groups as DO_NOT_USE for later segment determination where_sat = np.where(np.bitwise_and(gdq, dqflags.group['SATURATED'])) diff --git a/jwst/ramp_fitting/tests/test_ramp_fit.py b/jwst/ramp_fitting/tests/test_ramp_fit.py index 1313156eeaa..b7fb224aa79 100644 --- a/jwst/ramp_fitting/tests/test_ramp_fit.py +++ b/jwst/ramp_fitting/tests/test_ramp_fit.py @@ -13,7 +13,7 @@ JUMP_DET = dqflags.pixel["JUMP_DET"] SATURATED = dqflags.pixel["SATURATED"] NO_GAIN = dqflags.pixel["NO_GAIN_VALUE"] -UNDERSAMP = dqflags.pixel["UNDERSAMP"] +CHARGELOSS = dqflags.pixel["CHARGELOSS"] DELIM = "-" * 70 @@ -47,7 +47,7 @@ def test_drop_frames1_not_set(): def test_readnoise_variance(): - # test RN variance calculations for handling undersample_correction + # test RN variance calculations for handling charge_migration group_time = 10.6 model1, gdq_4d, rnoise, pixdq, err, gain = \ @@ -61,16 +61,16 @@ def test_readnoise_variance(): # Populate ramps with a variety of flags gdq_4d[:, 7, 1, 3] = JUMP_DET gdq_4d[:, 6:, 1, 2] = SATURATED - gdq_4d[:, 3:, 0, 3] = DO_NOT_USE + UNDERSAMP - gdq_4d[:, 7:, 2, 3] = DO_NOT_USE + UNDERSAMP + gdq_4d[:, 3:, 0, 3] = DO_NOT_USE + CHARGELOSS + gdq_4d[:, 7:, 2, 3] = DO_NOT_USE + CHARGELOSS gdq_4d[:, 3, 2, 2] = JUMP_DET gdq_4d[:, 6, 2, 2] = JUMP_DET - gdq_4d[:, 8:, 2, 2] = DO_NOT_USE + UNDERSAMP + gdq_4d[:, 8:, 2, 2] = DO_NOT_USE + CHARGELOSS gdq_4d[:, 8, 2, 2] += JUMP_DET gdq_4d[:, 0, 0, 0] = DO_NOT_USE + SATURATED gdq_4d[:, 1:, 0, 0] = SATURATED gdq_4d[:, 0, 0, 2] = SATURATED + DO_NOT_USE - gdq_4d[:, 1:, 0, 2] = SATURATED + DO_NOT_USE + UNDERSAMP + gdq_4d[:, 1:, 0, 2] = SATURATED + DO_NOT_USE + CHARGELOSS gdq_4d[:, 0:, 1, 0] = JUMP_DET var_r2, var_r3, var_r4 = compute_RN_variances(gdq_4d, readnoise_2d, gain_2d, group_time) diff --git a/jwst/regtest/test_niriss_image.py b/jwst/regtest/test_niriss_image.py index 5c3b2da37bc..e22f9fd3da8 100644 --- a/jwst/regtest/test_niriss_image.py +++ b/jwst/regtest/test_niriss_image.py @@ -1,5 +1,5 @@ """ Test for the detector1 pipeline using NIRISS image mode, starting with - an uncal file. The undersampling_correction and ramp fitting output + an uncal file. The charge_migration and ramp fitting output products are saved for comparisons for those two steps. """ @@ -18,8 +18,8 @@ def run_detector1(rtdata_module): # Run detector1 pipeline on an _uncal files args = ["calwebb_detector1", rtdata.input, - "--steps.undersampling_correction.skip=False", - "--steps.undersampling_correction.save_results=True", + "--steps.charge_migration.skip=False", + "--steps.charge_migration.save_results=True", "--steps.ramp_fit.save_results=True", "--steps.persistence.save_trapsfilled=False", ] @@ -28,7 +28,7 @@ def run_detector1(rtdata_module): @pytest.mark.bigdata -@pytest.mark.parametrize("suffix", ["undersampling_correction", "rate", "rateints"]) +@pytest.mark.parametrize("suffix", ["charge_migration", "rate", "rateints"]) def test_niriss_image_detector1(run_detector1, rtdata_module, fitsdiff_default_kwargs, suffix): """Regression test of detector1 pipeline performed on NIRISS imaging data. """ diff --git a/jwst/step.py b/jwst/step.py index 85c58ba7bd3..df270647d84 100644 --- a/jwst/step.py +++ b/jwst/step.py @@ -53,7 +53,8 @@ from .superbias.superbias_step import SuperBiasStep from .tso_photometry.tso_photometry_step import TSOPhotometryStep from .tweakreg.tweakreg_step import TweakRegStep -from .undersampling_correction.undersampling_correction_step import UndersamplingCorrectionStep +from .charge_migration.charge_migration_step import ChargeMigrationStep + from .wavecorr.wavecorr_step import WavecorrStep from .wfs_combine.wfs_combine_step import WfsCombineStep from .wfss_contam.wfss_contam_step import WfssContamStep @@ -115,7 +116,7 @@ "StraylightStep", "SuperBiasStep", "TSOPhotometryStep", - "UndersamplingCorrectionStep", + "ChargeMigrationStep", "TweakRegStep", "WavecorrStep", "WfsCombineStep", diff --git a/jwst/stpipe/integration.py b/jwst/stpipe/integration.py index 6fd651017cc..84daef34c21 100644 --- a/jwst/stpipe/integration.py +++ b/jwst/stpipe/integration.py @@ -86,7 +86,7 @@ def get_steps(): ("jwst.step.SuperBiasStep", 'superbias', False), ("jwst.step.TSOPhotometryStep", 'tso_photometry', False), ("jwst.step.TweakRegStep", 'tweakreg', False), - ("jwst.step.UndersamplingCorrectionStep", 'undersampling_correction', False), + ("jwst.step.ChargeMigrationStep", 'charge_migration', False), ("jwst.step.WavecorrStep", 'wavecorr', False), ("jwst.step.WfsCombineStep", 'calwebb_wfs-image3', False), ("jwst.step.WfssContamStep", 'wfss_contam', False),