Changed paths to reflect new naming structure for different processin…

…g levels.

Changed the way it generates the filename for the expected file being produced.

woest/apply_calc/SETTINGS.py

@@ -20,15 +20,16 @@
 LOTUS_DIR = f'{LOTUS_OUTPUT_PATH_BASE}/cals/nxpol1/'
 
 #LOCATION OF SCRIPTS
-SCRIPT_DIR = f'/gws/pw/j07/ncas_obs_vol1/amf/software/ncas-mobile-x-band-radar-1/apply_calc/'
+SCRIPT_DIR = f'/gws/pw/j07/ncas_obs_vol1/amf/software/ncas-mobile-x-band-radar-1/woest/apply_calc/'
 
-#Location of uncalibrated/unprocessed files
-INPUT_DIR = f'/gws/smf/j07/ncas_radar/data/ncas-mobile-x-band-radar-1/{PROJ_NAME}/cfradial/uncalib_v1/'
-OUTPUT_DIR = f'/gws/smf/j07/ncas_radar/data/ncas-mobile-x-band-radar-1/{PROJ_NAME}/cfradial/processed_v1/'
+#Location of uncalibrated files
+INPUT_DIR = f'/gws/smf/j07/ncas_radar/data/ncas-mobile-x-band-radar-1/{PROJ_NAME}/level1/'
+#Location of calibrated/processed files
+OUTPUT_DIR = f'/gws/smf/j07/ncas_radar/data/ncas-mobile-x-band-radar-1/{PROJ_NAME}/level2/'
 
 #Location of params files
-PARAMS_FILE = f'/home/users/lbennett/lrose/ingest_params/{PROJ_NAME}/RadxConvert.nxpol1_{PROJ_NAME}.processed' 
-PARAMS_FILE_RHI = f'/home/users/lbennett/lrose/ingest_params/{PROJ_NAME}/RadxConvert.nxpol1_{PROJ_NAME}_rhi.processed' 
+PARAMS_FILE = f'/home/users/lbennett/lrose/ingest_params/{PROJ_NAME}/RadxConvert.nxpol1_{PROJ_NAME}_sur.level2' 
+PARAMS_FILE_RHI = f'/home/users/lbennett/lrose/ingest_params/{PROJ_NAME}/RadxConvert.nxpol1_{PROJ_NAME}_rhi.level2' 
 
 CHUNK_HOURS=6
 

woest/apply_calc/apply_calc_chunk.py

@@ -39,13 +39,15 @@ def arg_parse_chunk():
 
 def loop_over_files(args):
 
-#    params_index = args.params_index[0]
-    #params_file = f'{SETTINGS.PARAMS_FILE}'
-    params_file = f'{SETTINGS.PARAMS_FILE_RHI}'
+    scan_geom = args.scan_geom[0]
+
+    if scan_geom=='sur':
+        params_file = f'{SETTINGS.PARAMS_FILE}'
+    elif scan_geom=='rhi':
+        params_file = f'{SETTINGS.PARAMS_FILE_RHI}'
+
     input_files = args.files
     print("input_files= ",input_files)
-    scan_geom = args.scan_geom[0]
-#    scan_type = args.scan_type[0]
     table = args.table_name[0]
 
     failure_count=0
@@ -57,13 +59,8 @@ def loop_over_files(args):
                                  f'{SETTINGS.EXIT_AFTER_N_FAILURES}')
 
         print("ncfile= ",ncfile)
-        fname = os.path.basename(ncfile)
-        fname=fname[:27] + 'mod-' + fname[27:]
-        ncdate = os.path.basename(ncfile).split('_')[2].replace('-','')
+        ncdate = os.path.basename(ncfile).split('_')[2]
 
-        YYYY=ncdate[0:4]
-        MM=ncdate[4:6]
-        DD=ncdate[6:8]
         date=ncdate[0:8]
 
         rh = DataBaseHandler(table_name=table)
@@ -78,7 +75,7 @@ def loop_over_files(args):
         # Remove previous results for this file
         rh.delete_result(identifier)
 
-        #Read input uncalibrated netcdf file and extract list of variables
+        #Read input level1 netcdf file and extract list of variables
         try:
             rad1=pyart.io.read(ncfile, delay_field_loading=True)
         except IOError:
@@ -99,10 +96,12 @@ def loop_over_files(args):
             failure_count += 1
             continue
 
-        #this line looks for the file generated from uncalib_v1 in calib_v1.
-        expected_file = f'{SETTINGS.OUTPUT_DIR}/{scan_geom}/{date}/{fname}'
+        #this line looks for the file generated from level1 data in the output level2 folder
+        outfiles=os.listdir(f'{SETTINGS.OUTPUT_DIR}/{scan_geom}/{date}/')
+        fname = [s for s in outfiles if ncdate in s]
+        expected_file = f'{SETTINGS.OUTPUT_DIR}/{scan_geom}/{date}/{fname[0]}'
         print("[INFO] Checking that the output file has been produced.")
-        #Read input uncalibrated netcdf file and extract list of variables
+        #Read outputted level2 netcdf file and extract list of variables
         try:
             rad2=pyart.io.read(expected_file, delay_field_loading=True)
         except IOError:
@@ -116,7 +115,7 @@ def loop_over_files(args):
         print(f'[INFO] Found expected file {expected_file}')
 
         print(f'[INFO] Checking that the output variables match those in the input files')
-        #Checks that the variables in the calibrated nc file are identical to the variables in the uncalibrated input files
+        #Checks that the variables in the level2 nc file are identical to the variables in the level1 input files
         #If not, create a failure identifier called bad_vars
         keys_not_found=[]
         for key in input_vars:

woest/apply_calc/apply_calc_time_series.py

@@ -93,7 +93,7 @@ def loop_over_days(args):
     files.sort()
     print(files)
 
-    lotus_logs=f'{SETTINGS.LOTUS_DIR}{today}'    
+    lotus_logs=f'{SETTINGS.LOTUS_DIR}{scan_geom}/{today}'    
     if not os.path.exists(lotus_logs):
         os.makedirs(lotus_logs)
 

woest/apply_calc/apply_calc_time_series_rhi.py

@@ -47,9 +47,7 @@ def loop_over_days(args):
     table = args.table_name[0]
 
     scan_geom = args.scan_geom[0]
-#    params_index = args.params_index[0]
     input_dir = os.path.join(SETTINGS.INPUT_DIR, scan_geom) 
-    #params_file = f'{SETTINGS.PARAMS_FILE}{params_index}'
 
     #How many files to process in each chunk. Approx 10 files each hour.   
     chunk_hours=SETTINGS.CHUNK_HOURS