Finding CDAT dependencies

[forsyth2]

This script will generate directly-called CDAT dependencies in e3sm_diags:

cd e3sm_diags
# git checkout cdat-migration-fy24

for diags_set in enso_diags qbo streamflow diurnal_cycle arm_diags tc_analysis annual_cycle_zonal_mean area_mean_time_series
do 
  echo ""
  echo ${diags_set}
  # https://anaconda.org/cdat/repo?sort=_name&sort_order=asc
  for cdat_package in cd77 cdat cdat_compute_graph cdat_info cdat_jupyter_vcdat cdms2 cdp cdtime cdutil cibots compute_graph django-allow-cidr dv3d e3sm_nex esgf-compute-api esgf-compute-cert esgf-search ezget genutil image-compare jupyter-vcdat jupyterlab-git lats libcdms lmoments mesalib myproxyclient output_viewer selenium_testlib testsrunner thermo toolchain vcdat vcs vcs-js vcsaddons vtk vtk-cdat wk xmgrace
  do
      git grep -n ${cdat_package} e3sm_diags/driver/${diags_set}_driver.py > /dev/null
      if [ $? == 0 ]; then
  	  echo ""
	  echo "git grep -n ${cdat_package} e3sm_diags/driver/${diags_set}_driver.py"
	  git grep -n ${cdat_package} e3sm_diags/driver/${diags_set}_driver.py
      fi
      git grep -n ${cdat_package} e3sm_diags/plot/cartopy/${diags_set}_plot.py > /dev/null
      if [ $? == 0 ]; then
	  echo ""
	  echo "git grep -n ${cdat_package} e3sm_diags/plot/cartopy/${diags_set}_plot.py"
	  git grep -n ${cdat_package} e3sm_diags/plot/cartopy/${diags_set}_plot.py
      fi
  done
done

The output is:

enso_diags

git grep -n cdms2 e3sm_diags/driver/enso_diags_driver.py
e3sm_diags/driver/enso_diags_driver.py:8:import cdms2
e3sm_diags/driver/enso_diags_driver.py:238:            with cdms2.open(mask_path) as f:

git grep -n cdutil e3sm_diags/driver/enso_diags_driver.py
e3sm_diags/driver/enso_diags_driver.py:9:import cdutil
e3sm_diags/driver/enso_diags_driver.py:96:        sst_avg = cdutil.averager(sst_nino, axis="xy")
e3sm_diags/driver/enso_diags_driver.py:98:        sst_avg_anomaly = cdutil.ANNUALCYCLE.departures(sst_avg)
e3sm_diags/driver/enso_diags_driver.py:127:    cdutil.setTimeBoundsMonthly(domain)
e3sm_diags/driver/enso_diags_driver.py:131:    anomaly = cdutil.ANNUALCYCLE.departures(domain)
e3sm_diags/driver/enso_diags_driver.py:135:    confidence_levels = cdutil.ANNUALCYCLE.departures(domain)[0, :, :](squeeze=1)
e3sm_diags/driver/enso_diags_driver.py:424:            test_avg = cdutil.averager(test_data_ts_regional, axis="xy")
e3sm_diags/driver/enso_diags_driver.py:425:            ref_avg = cdutil.averager(ref_data_ts_regional, axis="xy")
e3sm_diags/driver/enso_diags_driver.py:427:            y["test"] = cdutil.ANNUALCYCLE.departures(test_avg)
e3sm_diags/driver/enso_diags_driver.py:428:            y["ref"] = cdutil.ANNUALCYCLE.departures(ref_avg)

git grep -n cdutil e3sm_diags/plot/cartopy/enso_diags_plot.py
e3sm_diags/plot/cartopy/enso_diags_plot.py:6:import cdutil
e3sm_diags/plot/cartopy/enso_diags_plot.py:87:        domain = cdutil.region.domain(latitude=(-90.0, 90, "ccb"))

git grep -n genutil e3sm_diags/driver/enso_diags_driver.py
e3sm_diags/driver/enso_diags_driver.py:146:            # Uncomment the following line to use CDAT/genutil instead
e3sm_diags/driver/enso_diags_driver.py:148:            # slope, intercept = genutil.statistics.linearregression(dependent_var, x=independent_var)

qbo

git grep -n cdutil e3sm_diags/driver/qbo_driver.py
e3sm_diags/driver/qbo_driver.py:7:import cdutil
e3sm_diags/driver/qbo_driver.py:40:    data_lon_average = cdutil.averager(data_region, axis="x")
e3sm_diags/driver/qbo_driver.py:42:    data_lon_lat_average = cdutil.averager(data_lon_average, axis="y")
e3sm_diags/driver/qbo_driver.py:100:    data_lat_lon_average = cdutil.averager(data_region, axis="xy")

streamflow

git grep -n cdms2 e3sm_diags/driver/streamflow_driver.py
e3sm_diags/driver/streamflow_driver.py:7:import cdms2
e3sm_diags/driver/streamflow_driver.py:288:    if isinstance(area_upstream, cdms2.tvariable.TransientVariable):
e3sm_diags/driver/streamflow_driver.py:492:                if isinstance(monthly, cdms2.tvariable.TransientVariable):
e3sm_diags/driver/streamflow_driver.py:518:            if isinstance(monthly, cdms2.tvariable.TransientVariable):

git grep -n cdutil e3sm_diags/driver/streamflow_driver.py
e3sm_diags/driver/streamflow_driver.py:8:import cdutil
e3sm_diags/driver/streamflow_driver.py:227:        var_array = ref_data_ts(cdutil.region.domain(latitude=(-90.0, 90, "ccb")))
e3sm_diags/driver/streamflow_driver.py:270:        var_array = test_data_ts(cdutil.region.domain(latitude=(-90.0, 90, "ccb")))

git grep -n cdutil e3sm_diags/plot/cartopy/streamflow_plot.py
e3sm_diags/plot/cartopy/streamflow_plot.py:7:import cdutil
e3sm_diags/plot/cartopy/streamflow_plot.py:81:        domain = cdutil.region.domain(latitude=(-90.0, 90, "ccb"))
e3sm_diags/plot/cartopy/streamflow_plot.py:380:        domain = cdutil.region.domain(latitude=(-90.0, 90, "ccb"))

diurnal_cycle

git grep -n cdms2 e3sm_diags/driver/diurnal_cycle_driver.py
e3sm_diags/driver/diurnal_cycle_driver.py:6:import cdms2
e3sm_diags/driver/diurnal_cycle_driver.py:45:            with cdms2.open(mask_path) as f:

git grep -n cdutil e3sm_diags/plot/cartopy/diurnal_cycle_plot.py
e3sm_diags/plot/cartopy/diurnal_cycle_plot.py:7:import cdutil
e3sm_diags/plot/cartopy/diurnal_cycle_plot.py:110:        domain = cdutil.region.domain(latitude=(-90.0, 90, "ccb"))

arm_diags

git grep -n cdms2 e3sm_diags/driver/arm_diags_driver.py
e3sm_diags/driver/arm_diags_driver.py:8:import cdms2
e3sm_diags/driver/arm_diags_driver.py:103:                        ref_data = cdms2.open(ref_file)
e3sm_diags/driver/arm_diags_driver.py:208:                    ref_data = cdms2.open(ref_file)
e3sm_diags/driver/arm_diags_driver.py:316:                    ref_data = cdms2.open(ref_file)
e3sm_diags/driver/arm_diags_driver.py:393:            ref_data = cdms2.open(ref_file)
e3sm_diags/driver/arm_diags_driver.py:447:                ref_data = cdms2.open(ref_file)
e3sm_diags/driver/arm_diags_driver.py:516:                    ref_data = cdms2.open(ref_file)

tc_analysis

git grep -n cdms2 e3sm_diags/driver/tc_analysis_driver.py
e3sm_diags/driver/tc_analysis_driver.py:8:import cdms2
e3sm_diags/driver/tc_analysis_driver.py:77:    test_cyclones_hist = cdms2.open(test_cyclones_file)(
e3sm_diags/driver/tc_analysis_driver.py:84:    test_aew_hist = cdms2.open(test_aew_file)(
e3sm_diags/driver/tc_analysis_driver.py:113:        ref_cyclones_hist = cdms2.open(ref_cyclones_file)("density", squeeze=1)
e3sm_diags/driver/tc_analysis_driver.py:118:        ref_aew_hist = cdms2.open(ref_aew_file)("density", squeeze=1)
e3sm_diags/driver/tc_analysis_driver.py:133:        ref_cyclones_hist = cdms2.open(ref_cyclones_file)(
e3sm_diags/driver/tc_analysis_driver.py:137:        ref_aew_hist = cdms2.open(ref_aew_file)(
e3sm_diags/driver/tc_analysis_driver.py:176:    ocnfrac = cdms2.open(mask_path)("OCNFRAC", squeeze=1)
e3sm_diags/driver/tc_analysis_driver.py:281:    ocnfrac: cdms2.dataset.DatasetVariable,
e3sm_diags/driver/tc_analysis_driver.py:291:    :type ocnfrac: cdms2.dataset.DatasetVariable

annual_cycle_zonal_mean

git grep -n cdms2 e3sm_diags/driver/annual_cycle_zonal_mean_driver.py
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:7:from cdms2 import createAxis
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:19:    from cdms2.axis import TransientAxis
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:20:    from cdms2.tvariable import TransientVariable

git grep -n cdutil e3sm_diags/driver/annual_cycle_zonal_mean_driver.py
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:5:import cdutil
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:55:        test_ac_zonal_mean = cdutil.averager(test_ac, axis="x", weights="generate")
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:56:        test_ac_reg_zonal_mean = cdutil.averager(
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:73:                ref_ac_zonal_mean = cdutil.averager(
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:76:                ref_ac_reg_zonal_mean = cdutil.averager(
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:81:            ref_ac_zonal_mean = cdutil.averager(ref_ac, axis="x", weights="generate")
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:82:            ref_ac_reg_zonal_mean = cdutil.averager(
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:93:        #    ref_ac_zonal_mean = cdutil.averager(ref_ac, axis="x", weights="generate")
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:94:        #    ref_ac_reg_zonal_mean = cdutil.averager(

area_mean_time_series

git grep -n cdms2 e3sm_diags/driver/area_mean_time_series_driver.py
e3sm_diags/driver/area_mean_time_series_driver.py:8:import cdms2
e3sm_diags/driver/area_mean_time_series_driver.py:70:        with cdms2.open(mask_path) as f:

git grep -n cdutil e3sm_diags/driver/area_mean_time_series_driver.py
e3sm_diags/driver/area_mean_time_series_driver.py:9:import cdutil
e3sm_diags/driver/area_mean_time_series_driver.py:97:            test_domain = cdutil.averager(test_domain, axis="xy")
e3sm_diags/driver/area_mean_time_series_driver.py:98:            cdutil.setTimeBoundsMonthly(test_domain)
e3sm_diags/driver/area_mean_time_series_driver.py:99:            test_domain_year = cdutil.YEAR(test_domain)
e3sm_diags/driver/area_mean_time_series_driver.py:100:            # add back attributes since they got lost after applying cdutil.YEAR
e3sm_diags/driver/area_mean_time_series_driver.py:123:                    ref_domain = cdutil.averager(ref_domain, axis="xy")
e3sm_diags/driver/area_mean_time_series_driver.py:124:                    cdutil.setTimeBoundsMonthly(ref_domain)
e3sm_diags/driver/area_mean_time_series_driver.py:133:                    ref_domain_year = cdutil.YEAR(ref_domain)

[forsyth2]

@tomvothecoder FYI ^

[tomvothecoder]

Thanks Ryan!

[forsyth2]

@tomvothecoder I made another version of the script here, which looks for any CDAT dependency anywhere in e3sm_diags:

cd e3sm_diags
# git checkout cdat-migration-fy24

# https://anaconda.org/cdat/repo?sort=_name&sort_order=asc
for cdat_package in cd77 cdat cdat_compute_graph cdat_info cdat_jupyter_vcdat cdms2 cdp cdtime cdutil cibots compute_graph django-allow-cidr dv3d e3sm_nex esgf-compute-api esgf-compute-cert esgf-search ezget genutil image-compare jupyter-vcdat jupyterlab-git lats libcdms lmoments mesalib myproxyclient output_viewer selenium_testlib testsrunner thermo toolchain vcdat vcs vcs-js vcsaddons vtk vtk-cdat wk xmgrace
do
    git grep -n ${cdat_package} e3sm_diags/* > /dev/null
    if [ $? == 0 ]; then
	echo ""
	echo "git grep -n ${cdat_package} e3sm_diags/*"
	git grep -n ${cdat_package} e3sm_diags/*
    fi
done

Output:

git grep -n cdat e3sm_diags/*
e3sm_diags/driver/aerosol_aeronet_driver.py:9:import xcdat as xc
e3sm_diags/driver/meridional_mean_2d_driver.py:7:import xcdat as xc
e3sm_diags/driver/utils/climo_xr.py:12:import xcdat as xc
e3sm_diags/driver/utils/dataset_xr.py:22:import xcdat as xc
e3sm_diags/driver/utils/regrid.py:6:import xcdat as xc
e3sm_diags/driver/utils/regrid.py:386:    .. [1] https://xcdat.readthedocs.io/en/stable/generated/xarray.Dataset.regridder.horizontal.html
e3sm_diags/driver/zonal_mean_2d_driver.py:5:import xcdat as xc  # noqa: F401
e3sm_diags/driver/zonal_mean_xy_driver.py:6:import xcdat as xc
e3sm_diags/metrics/metrics.py:8:import xcdat as xc
e3sm_diags/metrics/metrics.py:262:    * Related issue: https://github.com/xCDAT/xcdat/issues/596
e3sm_diags/metrics/metrics.py:263:    * Source: https://github.com/xCDAT/xcdat/blob/main/xcdat/spatial.py#L479-L495C16
e3sm_diags/plot/lat_lon_plot.py:9:import xcdat as xc
e3sm_diags/plot/meridional_mean_2d_plot.py:5:import xcdat as xc
e3sm_diags/plot/polar_plot.py:8:import xcdat as xc
e3sm_diags/plot/utils.py:11:import xcdat as xc
e3sm_diags/plot/zonal_mean_2d_plot.py:6:import xcdat as xc
e3sm_diags/plot/zonal_mean_xy_plot.py:3:import xcdat as xc

git grep -n cdms2 e3sm_diags/*
e3sm_diags/__init__.py:4:# import shapely here (before any esmpy imports via cdms2) to prevent a
e3sm_diags/__init__.py:12:# Disable MPI in cdms2, which is not currently supported by E3SM-unified
e3sm_diags/derivations/acme.py:12:    from cdms2.axis import FileAxis
e3sm_diags/derivations/acme.py:13:    from cdms2.fvariable import FileVariable
e3sm_diags/derivations/acme.py:2097:        var (cdms2.TransientVariable): The input burden in kg/m2.
e3sm_diags/derivations/acme.py:2100:        burden (cdms2.TransientVariable): The output burden in 1e-6 kg/m2.
e3sm_diags/derivations/acme.py:2128:        var (cdms2.TransientVariable): The input mass in kg/kg.
e3sm_diags/derivations/acme.py:2131:        cdms2.TransientVariable: The aerosol mass concentration in 1e-12 kg/kg units.
e3sm_diags/derivations/acme.py:2195:        cdnc (cdms2.TransientVariable): Cloud droplet number concentration in 1/m3.
e3sm_diags/derivations/acme.py:2196:        lcc (cdms2.TransientVariable): Liquid cloud fraction.
e3sm_diags/derivations/acme.py:2199:        var (cdms2.TransientVariable): In-cloud cdnc at cloud top in 1/cm3.
e3sm_diags/derivations/acme.py:2212:        cdnc (cdms2.TransientVariable): Cloud droplet number concentration in 1/m3.
e3sm_diags/derivations/acme.py:2215:        var (cdms2.TransientVariable): In-grid cdnc at cloud top in 1/cm3.
e3sm_diags/derivations/acme.py:2228:        icnc (cdms2.TransientVariable): ice crystal number concentration in 1/m3.
e3sm_diags/derivations/acme.py:2229:        icc (cdms2.TransientVariable): ice cloud fraction.
e3sm_diags/derivations/acme.py:2232:        var (cdms2.TransientVariable): In-cloud cdnc at cloud top in 1/cm3.
e3sm_diags/derivations/acme.py:2245:        icnc (cdms2.TransientVariable): Cloud crystal number concentration in 1/m3.
e3sm_diags/derivations/acme.py:2248:        var (cdms2.TransientVariable): In-grid icnc at cloud top in 1/cm3.
e3sm_diags/derivations/acme.py:2261:        lwp (cdms2.TransientVariable): Liquid water path in kg/m2.
e3sm_diags/derivations/acme.py:2262:        lcc (cdms2.TransientVariable): Liquid cloud fraction.
e3sm_diags/derivations/acme.py:2265:        cdms2.TransientVariable: In-cloud liquid water path in g/cm3.
e3sm_diags/derivations/acme.py:2278:                lwp (cdms2.TransientVariable): Liquid Water Path (LWP) value.
e3sm_diags/derivations/acme.py:2281:           cdms2.TransientVariable: Grid-mean-cloud LWP in g/cm3.
e3sm_diags/derivations/acme.py:2294:        iwp (cdms2.TransientVariable): Ice water path in kg/m2.
e3sm_diags/derivations/acme.py:2295:        icc (cdms2.TransientVariable): Ice cloud fraction.
e3sm_diags/derivations/acme.py:2298:        cdms2.TransientVariable: In-cloud IWP in g/cm3.
e3sm_diags/derivations/acme.py:2311:        iwp (cdms2.TransientVariable): Ice water path in kg/m2.
e3sm_diags/derivations/acme.py:2314:        cdms2.TransientVariable: In-grid IWP in g/cm3.
e3sm_diags/derivations/acme.py:2342:        fsnt (cdms2.TransientVariable): The incoming sw radiation at the top of the atmosphere.
e3sm_diags/derivations/acme.py:2343:        flnt (cdms2.TransientVariable): The outgoing lw radiation at the top of the atmosphere.
e3sm_diags/derivations/acme.py:2346:        var (cdms2.TransientVariable): The ERFtot which represents the total erf.
e3sm_diags/derivations/acme.py:2361:        fsnt (cdms2.TransientVariable): Net solar flux at the top of the atmosphere.
e3sm_diags/derivations/acme.py:2362:        flnt (cdms2.TransientVariable): Net longwave flux at the top of the atmosphere.
e3sm_diags/derivations/acme.py:2363:        fsnt_d1 (cdms2.TransientVariable): fsnt without aerosols.
e3sm_diags/derivations/acme.py:2364:        flnt_d1 (cdms2.TransientVariable): flnt without aerosols.
e3sm_diags/derivations/acme.py:2367:        var (cdms2.TransientVariable): ERFari (aka, direct effect) in W/m2.
e3sm_diags/derivations/acme.py:2382:        fsnt_d1 (cdms2.TransientVariable): Downward shortwave radiation toa without aerosols.
e3sm_diags/derivations/acme.py:2383:        flnt_d1 (cdms2.TransientVariable): Upward longwave radiation toa without aerosols.
e3sm_diags/derivations/acme.py:2384:        fsntc_d1 (cdms2.TransientVariable): fsnt_d1 without clouds.
e3sm_diags/derivations/acme.py:2385:        flntc_d1 (cdms2.TransientVariable): flnt_d1 without clouds.
e3sm_diags/derivations/acme.py:2388:        var (cdms2.TransientVariable): ERFaci (aka, indirect effect) in W/m2.
e3sm_diags/derivations/acme.py:2403:        fsntc_d1 (cdms2.TransientVariable): Downward solar radiation at the top of the atmosphere
e3sm_diags/derivations/acme.py:2405:        flntc_d1 (cdms2.TransientVariable): Upward longwave radiation at the top of the atmosphere
e3sm_diags/derivations/acme.py:2409:        var (cdms2.TransientVariable): ERFres (aka, surface effect) in W/m2.
e3sm_diags/derivations/formulas.py:457:    - CDAT/cdms2 replaces these values with the floats from `rsutcs`, but they
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:7:from cdms2 import createAxis
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:19:    from cdms2.axis import TransientAxis
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:20:    from cdms2.tvariable import TransientVariable
e3sm_diags/driver/area_mean_time_series_driver.py:8:import cdms2
e3sm_diags/driver/area_mean_time_series_driver.py:70:        with cdms2.open(mask_path) as f:
e3sm_diags/driver/arm_diags_driver.py:8:import cdms2
e3sm_diags/driver/arm_diags_driver.py:103:                        ref_data = cdms2.open(ref_file)
e3sm_diags/driver/arm_diags_driver.py:208:                    ref_data = cdms2.open(ref_file)
e3sm_diags/driver/arm_diags_driver.py:316:                    ref_data = cdms2.open(ref_file)
e3sm_diags/driver/arm_diags_driver.py:393:            ref_data = cdms2.open(ref_file)
e3sm_diags/driver/arm_diags_driver.py:447:                ref_data = cdms2.open(ref_file)
e3sm_diags/driver/arm_diags_driver.py:516:                    ref_data = cdms2.open(ref_file)
e3sm_diags/driver/diurnal_cycle_driver.py:6:import cdms2
e3sm_diags/driver/diurnal_cycle_driver.py:45:            with cdms2.open(mask_path) as f:
e3sm_diags/driver/enso_diags_driver.py:8:import cdms2
e3sm_diags/driver/enso_diags_driver.py:238:            with cdms2.open(mask_path) as f:
e3sm_diags/driver/streamflow_driver.py:7:import cdms2
e3sm_diags/driver/streamflow_driver.py:288:    if isinstance(area_upstream, cdms2.tvariable.TransientVariable):
e3sm_diags/driver/streamflow_driver.py:492:                if isinstance(monthly, cdms2.tvariable.TransientVariable):
e3sm_diags/driver/streamflow_driver.py:518:            if isinstance(monthly, cdms2.tvariable.TransientVariable):
e3sm_diags/driver/tc_analysis_driver.py:8:import cdms2
e3sm_diags/driver/tc_analysis_driver.py:77:    test_cyclones_hist = cdms2.open(test_cyclones_file)(
e3sm_diags/driver/tc_analysis_driver.py:84:    test_aew_hist = cdms2.open(test_aew_file)(
e3sm_diags/driver/tc_analysis_driver.py:113:        ref_cyclones_hist = cdms2.open(ref_cyclones_file)("density", squeeze=1)
e3sm_diags/driver/tc_analysis_driver.py:118:        ref_aew_hist = cdms2.open(ref_aew_file)("density", squeeze=1)
e3sm_diags/driver/tc_analysis_driver.py:133:        ref_cyclones_hist = cdms2.open(ref_cyclones_file)(
e3sm_diags/driver/tc_analysis_driver.py:137:        ref_aew_hist = cdms2.open(ref_aew_file)(
e3sm_diags/driver/tc_analysis_driver.py:176:    ocnfrac = cdms2.open(mask_path)("OCNFRAC", squeeze=1)
e3sm_diags/driver/tc_analysis_driver.py:281:    ocnfrac: cdms2.dataset.DatasetVariable,
e3sm_diags/driver/tc_analysis_driver.py:291:    :type ocnfrac: cdms2.dataset.DatasetVariable
e3sm_diags/driver/utils/climo.py:3:`cdms2.TransientVariable`.
e3sm_diags/driver/utils/climo.py:8:import cdms2
e3sm_diags/driver/utils/climo.py:87:    trans_var = cdms2.createVariable(climo)(squeeze=1)
e3sm_diags/driver/utils/dataset.py:12:import cdms2
e3sm_diags/driver/utils/dataset.py:183:        fin = cdms2.open(file_path)
e3sm_diags/driver/utils/dataset.py:233:        with cdms2.open(filename) as f:
e3sm_diags/driver/utils/dataset.py:343:        with cdms2.open(filename) as data_file:
e3sm_diags/driver/utils/dataset.py:357:                    # Get the variables as cdms2.TransientVariables.
e3sm_diags/driver/utils/dataset.py:358:                    # Ex: variables is [PRECC, PRECL], where both are cdms2.TransientVariables.
e3sm_diags/driver/utils/dataset.py:428:        from the data_file as cdms2.TransientVariables.
e3sm_diags/driver/utils/dataset.py:475:                # Open the files of the variables and get the cdms2.TransientVariables.
e3sm_diags/driver/utils/dataset.py:635:        from files in data_path as cdms2.TransientVariables.
e3sm_diags/driver/utils/dataset.py:695:            # with cdms2.open(fnm) as f:
e3sm_diags/driver/utils/dataset.py:699:            fin = cdms2.open(fnm)
e3sm_diags/driver/utils/general.py:7:import cdms2
e3sm_diags/driver/utils/general.py:46:    time2 = cdms2.createAxis(var_time)
e3sm_diags/driver/utils/general.py:266:            cdms2.setNetcdfDeflateFlag(1)
e3sm_diags/driver/utils/general.py:268:            cdms2.setNetcdfDeflateLevelFlag(0)
e3sm_diags/driver/utils/general.py:269:            cdms2.setNetcdfShuffleFlag(0)
e3sm_diags/driver/utils/general.py:270:            cdms2.setCompressionWarnings(0)  # Turn off warning messages
e3sm_diags/driver/utils/general.py:282:            with cdms2.open(test_pth, "w+") as file_test:
e3sm_diags/driver/utils/general.py:297:        cdms2.setNetcdfDeflateFlag(1)
e3sm_diags/driver/utils/general.py:299:        cdms2.setNetcdfDeflateLevelFlag(0)
e3sm_diags/driver/utils/general.py:300:        cdms2.setNetcdfShuffleFlag(0)
e3sm_diags/driver/utils/general.py:301:        cdms2.setCompressionWarnings(0)  # Turn off warning messages
e3sm_diags/driver/utils/general.py:315:        with cdms2.open(test_pth, cdms_arg) as file_test:
e3sm_diags/driver/utils/general.py:323:            with cdms2.open(ref_pth, cdms_arg) as file_ref:
e3sm_diags/driver/utils/general.py:331:                with cdms2.open(diff_pth, cdms_arg) as file_diff:
e3sm_diags/driver/utils/regrid.py:152:    - https://cdms.readthedocs.io/en/latest/_modules/cdms2/avariable.html#AbstractVariable.getLevel
e3sm_diags/driver/utils/regrid.py:153:    - https://cdms.readthedocs.io/en/latest/_modules/cdms2/axis.html#AbstractAxis.isLevel
e3sm_diags/driver/utils/regrid.py:203:          - "conservative_normed" -- equivalent to "conservative" in cdms2 ESMF
e3sm_diags/driver/utils/regrid.py:223:    # NOTE: this is equivalent to "conservative" in cdms2 ESMF. If
e3sm_diags/e3sm_diags_vars.py:12:import cdms2
e3sm_diags/e3sm_diags_vars.py:49:    with cdms2.open(path) as f:

git grep -n cdp e3sm_diags/*
e3sm_diags/parameter/core_parameter.py:197:        It is based on cdp.cdp_parameter.CDPParameter.__add__().

git grep -n cdutil e3sm_diags/*
e3sm_diags/derivations/default_regions.py:5:import cdutil
e3sm_diags/derivations/default_regions.py:8:    "NHEX": {"domain": cdutil.region.domain(latitude=(30.0, 90, "ccb"))},
e3sm_diags/derivations/default_regions.py:9:    "SHEX": {"domain": cdutil.region.domain(latitude=(-90.0, -30, "ccb"))},
e3sm_diags/derivations/default_regions.py:10:    "TROPICS": {"domain": cdutil.region.domain(latitude=(-30.0, 30, "ccb"))},
e3sm_diags/derivations/default_regions.py:12:    "TRMM_region": {"domain": cdutil.region.domain(latitude=(-38.0, 38, "ccb"))},
e3sm_diags/derivations/default_regions.py:13:    "90S50S": {"domain": cdutil.region.domain(latitude=(-90.0, -50, "ccb"))},
e3sm_diags/derivations/default_regions.py:14:    "50S20S": {"domain": cdutil.region.domain(latitude=(-50.0, -20, "ccb"))},
e3sm_diags/derivations/default_regions.py:15:    "20S20N": {"domain": cdutil.region.domain(latitude=(-20.0, 20, "ccb"))},
e3sm_diags/derivations/default_regions.py:16:    "50S50N": {"domain": cdutil.region.domain(latitude=(-50.0, 50, "ccb"))},
e3sm_diags/derivations/default_regions.py:17:    "5S5N": {"domain": cdutil.region.domain(latitude=(-5.0, 5, "ccb"))},
e3sm_diags/derivations/default_regions.py:18:    "20N50N": {"domain": cdutil.region.domain(latitude=(20.0, 50, "ccb"))},
e3sm_diags/derivations/default_regions.py:19:    "50N90N": {"domain": cdutil.region.domain(latitude=(50.0, 90, "ccb"))},
e3sm_diags/derivations/default_regions.py:20:    "60S90N": {"domain": cdutil.region.domain(latitude=(-60.0, 90, "ccb"))},
e3sm_diags/derivations/default_regions.py:21:    "60S60N": {"domain": cdutil.region.domain(latitude=(-60.0, 60, "ccb"))},
e3sm_diags/derivations/default_regions.py:22:    "75S75N": {"domain": cdutil.region.domain(latitude=(-75.0, 75, "ccb"))},
e3sm_diags/derivations/default_regions.py:34:        "domain": cdutil.region.domain(latitude=(-60.0, 90, "ccb")),
e3sm_diags/derivations/default_regions.py:38:        "domain": cdutil.region.domain(latitude=(-30.0, 30, "ccb")),
e3sm_diags/derivations/default_regions.py:42:        "domain": cdutil.region.domain(latitude=(30.0, 90, "ccb")),
e3sm_diags/derivations/default_regions.py:46:        "domain": cdutil.region.domain(latitude=(-90.0, -30, "ccb")),
e3sm_diags/derivations/default_regions.py:50:        "domain": cdutil.region.domain(latitude=(-30.0, 30, "ccb")),
e3sm_diags/derivations/default_regions.py:54:        "domain": cdutil.region.domain(latitude=(30.0, 90, "ccb")),
e3sm_diags/derivations/default_regions.py:58:        "domain": cdutil.region.domain(latitude=(-90.0, -30, "ccb")),
e3sm_diags/derivations/default_regions.py:61:    "polar_N": {"domain": cdutil.region.domain(latitude=(50.0, 90.0, "ccb"))},
e3sm_diags/derivations/default_regions.py:62:    "polar_S": {"domain": cdutil.region.domain(latitude=(-90.0, -55.0, "ccb"))},
e3sm_diags/derivations/default_regions.py:68:        "domain": cdutil.region.domain(
e3sm_diags/derivations/default_regions.py:73:        "domain": cdutil.region.domain(
e3sm_diags/derivations/default_regions.py:78:        "domain": cdutil.region.domain(
e3sm_diags/derivations/default_regions.py:83:        "domain": cdutil.region.domain(
e3sm_diags/derivations/default_regions.py:88:        "domain": cdutil.region.domain(
e3sm_diags/derivations/default_regions.py:95:        "domain": cdutil.region.domain(
e3sm_diags/derivations/default_regions.py:101:        "domain": cdutil.region.domain(
e3sm_diags/derivations/default_regions.py:107:        "domain": cdutil.region.domain(
e3sm_diags/derivations/default_regions.py:113:        "domain": cdutil.region.domain(
e3sm_diags/derivations/default_regions.py:119:        "domain": cdutil.region.domain(
e3sm_diags/derivations/default_regions.py:125:        "domain": cdutil.region.domain(
e3sm_diags/derivations/default_regions.py:131:        "domain": cdutil.region.domain(
e3sm_diags/derivations/default_regions.py:137:        "domain": cdutil.region.domain(
e3sm_diags/derivations/default_regions.py:142:        "domain": cdutil.region.domain(
e3sm_diags/derivations/default_regions.py:147:        "domain": cdutil.region.domain(
e3sm_diags/derivations/default_regions.py:153:        "domain": cdutil.region.domain(
e3sm_diags/derivations/default_regions.py:158:        "domain": cdutil.region.domain(
e3sm_diags/derivations/default_regions.py:163:        "domain": cdutil.region.domain(
e3sm_diags/derivations/default_regions.py:168:    # 'CONUS_RRM': {'domain': cdutil.region.domain(latitude=(20., 50., 'ccb'), longitude=(-125., -65., 'ccb'))},For RRM dataset, negative value won't work
e3sm_diags/derivations/default_regions.py:170:        "domain": cdutil.region.domain(
e3sm_diags/derivations/default_regions.py:175:    "DEBUG": {"domain": cdutil.region.domain(latitude=(-2.0, 2, "ccb"))},
e3sm_diags/driver/aerosol_budget_driver.py:14:import cdutil
e3sm_diags/driver/aerosol_budget_driver.py:53:        pressure_levs = cdutil.vertical.reconstructPressureFromHybrid(
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:5:import cdutil
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:55:        test_ac_zonal_mean = cdutil.averager(test_ac, axis="x", weights="generate")
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:56:        test_ac_reg_zonal_mean = cdutil.averager(
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:73:                ref_ac_zonal_mean = cdutil.averager(
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:76:                ref_ac_reg_zonal_mean = cdutil.averager(
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:81:            ref_ac_zonal_mean = cdutil.averager(ref_ac, axis="x", weights="generate")
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:82:            ref_ac_reg_zonal_mean = cdutil.averager(
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:93:        #    ref_ac_zonal_mean = cdutil.averager(ref_ac, axis="x", weights="generate")
e3sm_diags/driver/annual_cycle_zonal_mean_driver.py:94:        #    ref_ac_reg_zonal_mean = cdutil.averager(
e3sm_diags/driver/area_mean_time_series_driver.py:9:import cdutil
e3sm_diags/driver/area_mean_time_series_driver.py:97:            test_domain = cdutil.averager(test_domain, axis="xy")
e3sm_diags/driver/area_mean_time_series_driver.py:98:            cdutil.setTimeBoundsMonthly(test_domain)
e3sm_diags/driver/area_mean_time_series_driver.py:99:            test_domain_year = cdutil.YEAR(test_domain)
e3sm_diags/driver/area_mean_time_series_driver.py:100:            # add back attributes since they got lost after applying cdutil.YEAR
e3sm_diags/driver/area_mean_time_series_driver.py:123:                    ref_domain = cdutil.averager(ref_domain, axis="xy")
e3sm_diags/driver/area_mean_time_series_driver.py:124:                    cdutil.setTimeBoundsMonthly(ref_domain)
e3sm_diags/driver/area_mean_time_series_driver.py:133:                    ref_domain_year = cdutil.YEAR(ref_domain)
e3sm_diags/driver/enso_diags_driver.py:9:import cdutil
e3sm_diags/driver/enso_diags_driver.py:96:        sst_avg = cdutil.averager(sst_nino, axis="xy")
e3sm_diags/driver/enso_diags_driver.py:98:        sst_avg_anomaly = cdutil.ANNUALCYCLE.departures(sst_avg)
e3sm_diags/driver/enso_diags_driver.py:127:    cdutil.setTimeBoundsMonthly(domain)
e3sm_diags/driver/enso_diags_driver.py:131:    anomaly = cdutil.ANNUALCYCLE.departures(domain)
e3sm_diags/driver/enso_diags_driver.py:135:    confidence_levels = cdutil.ANNUALCYCLE.departures(domain)[0, :, :](squeeze=1)
e3sm_diags/driver/enso_diags_driver.py:424:            test_avg = cdutil.averager(test_data_ts_regional, axis="xy")
e3sm_diags/driver/enso_diags_driver.py:425:            ref_avg = cdutil.averager(ref_data_ts_regional, axis="xy")
e3sm_diags/driver/enso_diags_driver.py:427:            y["test"] = cdutil.ANNUALCYCLE.departures(test_avg)
e3sm_diags/driver/enso_diags_driver.py:428:            y["ref"] = cdutil.ANNUALCYCLE.departures(ref_avg)
e3sm_diags/driver/mp_partition_driver.py:86:    # landfrac = test_data.get_timeseries_variable("LANDFRAC")(cdutil.region.domain(latitude=(-70.0, -30, "ccb")))
e3sm_diags/driver/mp_partition_driver.py:87:    # temp = test_data.get_timeseries_variable("T")(cdutil.region.domain(latitude=(-70.0, -30, "ccb")))
e3sm_diags/driver/mp_partition_driver.py:88:    # cice = test_data.get_timeseries_variable("CLDICE")(cdutil.region.domain(latitude=(-70.0, -30, "ccb")))
e3sm_diags/driver/mp_partition_driver.py:89:    # cliq = test_data.get_timeseries_variable("CLDLIQ")(cdutil.region.domain(latitude=(-70.0, -30, "ccb")))
e3sm_diags/driver/mp_partition_driver.py:133:        #    cdutil.region.domain(latitude=(-70.0, -30, "ccb"))
e3sm_diags/driver/mp_partition_driver.py:136:        #    cdutil.region.domain(latitude=(-70.0, -30, "ccb"))
e3sm_diags/driver/mp_partition_driver.py:139:        #    cdutil.region.domain(latitude=(-70.0, -30, "ccb"))
e3sm_diags/driver/mp_partition_driver.py:142:        #    cdutil.region.domain(latitude=(-70.0, -30, "ccb"))
e3sm_diags/driver/qbo_driver.py:7:import cdutil
e3sm_diags/driver/qbo_driver.py:40:    data_lon_average = cdutil.averager(data_region, axis="x")
e3sm_diags/driver/qbo_driver.py:42:    data_lon_lat_average = cdutil.averager(data_lon_average, axis="y")
e3sm_diags/driver/qbo_driver.py:100:    data_lat_lon_average = cdutil.averager(data_region, axis="xy")
e3sm_diags/driver/streamflow_driver.py:8:import cdutil
e3sm_diags/driver/streamflow_driver.py:227:        var_array = ref_data_ts(cdutil.region.domain(latitude=(-90.0, 90, "ccb")))
e3sm_diags/driver/streamflow_driver.py:270:        var_array = test_data_ts(cdutil.region.domain(latitude=(-90.0, 90, "ccb")))
e3sm_diags/driver/utils/general.py:8:import cdutil
e3sm_diags/driver/utils/general.py:135:    levels_orig = cdutil.vertical.reconstructPressureFromHybrid(ps, hyam, hybm, p0)
e3sm_diags/driver/utils/general.py:141:    var_p = cdutil.vertical.logLinearInterpolation(
e3sm_diags/driver/utils/general.py:166:    var_p = cdutil.vertical.logLinearInterpolation(
e3sm_diags/driver/utils/regrid.py:121:    Replaces `cdutil.variable.TransientVariable.getLevel()`.
e3sm_diags/driver/utils/regrid.py:621:    and `cdutil.vertical.reconstructPressureFromHybrid()`.
e3sm_diags/metrics/__init__.py:1:import cdutil
e3sm_diags/metrics/__init__.py:27:    return cdutil.averager(variable, axis=axis, weights="generate")
e3sm_diags/plot/cartopy/diurnal_cycle_plot.py:7:import cdutil
e3sm_diags/plot/cartopy/diurnal_cycle_plot.py:110:        domain = cdutil.region.domain(latitude=(-90.0, 90, "ccb"))
e3sm_diags/plot/cartopy/enso_diags_plot.py:6:import cdutil
e3sm_diags/plot/cartopy/enso_diags_plot.py:87:        domain = cdutil.region.domain(latitude=(-90.0, 90, "ccb"))
e3sm_diags/plot/cartopy/streamflow_plot.py:7:import cdutil
e3sm_diags/plot/cartopy/streamflow_plot.py:81:        domain = cdutil.region.domain(latitude=(-90.0, 90, "ccb"))
e3sm_diags/plot/cartopy/streamflow_plot.py:380:        domain = cdutil.region.domain(latitude=(-90.0, 90, "ccb"))

git grep -n genutil e3sm_diags/*
e3sm_diags/derivations/acme.py:9:from genutil import udunits
e3sm_diags/derivations/utils.py:6:from genutil import udunits
e3sm_diags/derivations/utils.py:85:        # FIXME: Replace genutil.udunits module.
e3sm_diags/driver/enso_diags_driver.py:146:            # Uncomment the following line to use CDAT/genutil instead
e3sm_diags/driver/enso_diags_driver.py:148:            # slope, intercept = genutil.statistics.linearregression(dependent_var, x=independent_var)
e3sm_diags/driver/utils/general.py:9:import genutil
e3sm_diags/driver/utils/general.py:157:    var, levels_orig = genutil.grower(var, levels_orig)
e3sm_diags/metrics/__init__.py:2:import genutil
e3sm_diags/metrics/__init__.py:14:            genutil.statistics.correlation(model, obs, axis=axis, weights="generate")
e3sm_diags/metrics/__init__.py:37:        rmse = float(genutil.statistics.rms(model, obs, axis=axis, weights="generate"))
e3sm_diags/metrics/__init__.py:46:        std = float(genutil.statistics.std(variable, axis=axis, weights="generate"))

git grep -n output_viewer e3sm_diags/*
e3sm_diags/viewer/core_viewer.py:4:from output_viewer.build import build_page, build_viewer
e3sm_diags/viewer/core_viewer.py:5:from output_viewer.index import (
e3sm_diags/viewer/core_viewer.py:12:from output_viewer.utils import rechmod

git grep -n vcs e3sm_diags/*
e3sm_diags/plot/__init__.py:73:    if not colormap.endswith(".rgb"):  # predefined vcs/mpl colormap

[forsyth2]

@tomvothecoder Many of the CDAT dependencies come in via calls to functions defined elsewhere. There are several ways to visualize such dependencies, which may prove more useful than this text-based approach.

Someone mentioned at the US RSE conference that PyCharm has a dependency graph feature. I believe that's this: https://www.jetbrains.com/help/pycharm/viewing-model-dependency-diagram.html

Since we use VS Code, perhaps https://marketplace.visualstudio.com/items?itemName=sz-p.dependencygraph would be a better plug-in to try out. (Assuming we don't need any special approvals to install third-party plug-ins).

[forsyth2]

I created a conda dev environment without CDAT so I could see if our pre-commit tools could find any remaining CDAT dependencies, but apparently that doesn't provide any useful information. (I was hoping the linter would produce output like "this function isn't defined" [because the CDAT dependency isn't installed]).

For reference, I wrote conda-env/dev-nocdat.yml:

# Conda development environment for testing local source code changes to `e3sm_diags` before merging them to production (`master` branch).
name: e3sm_diags_dev
channels:
  - conda-forge
  - defaults
dependencies:
  # Base
  # =======================
  - python >=3.9
  - pip
  - beautifulsoup4
  - cartopy >=0.17.0
  - cartopy_offlinedata
  # - cdms2 3.1.5
  # - cdutil 8.2.1
  - dask
  - esmpy >=8.4.0
  # - genutil 8.2.1
  - lxml
  - mache >=0.15.0
  - matplotlib-base
  - netcdf4
  # - output_viewer >=1.3.0
  - numpy >=1.23.0
  - shapely >=2.0.0,<3.0.0
  - xarray >=2023.02.0
  - xcdat >=0.6.0
  - xesmf >=0.7.0
  - xskillscore >=0.0.20
  # Testing
  # =======================
  - scipy
  - pytest
  - pytest-cov
  # Documentation
  # =======================
  - sphinx
  - sphinx_rtd_theme
  - sphinx-multiversion
  # Quality Assurance Tools
  # =======================
  # Run `pre-commit autoupdate` to get the latest pinned versions of 'rev' in
  # `.pre-commit.config.yaml`, then update the pinned versions here.
  - black=23.9.1
  - flake8=6.1.0
  - flake8-isort=6.1.0
  - isort=5.12.0
  - mypy=1.5.1
  - pre-commit >=3.0.0
  - types-PyYAML >=6.0.0
  # Developer Tools
  # =======================
  - tbump=6.9.0
  - ipykernel

Then:

$ mamba clean --all
$ mamba env create -f conda-env/dev-nocdat.yml -n e3sm_diags_nocdat
$ conda activate e3sm_diags_nocdat
$ pip install . # On copy of the cdat-migration-fy24 branch
$ pre-commit run --all-files
# After putting a `noqa` line for formatting in the aerosol_budget_driver, I got everything passing:
pre-commit run --all-files
trim trailing whitespace.................................................Passed
fix end of files.........................................................Passed
check yaml...............................................................Passed
black....................................................................Passed
isort....................................................................Passed
flake8...................................................................Passed
mypy.....................................................................Passed