config and launch

tools/demo.py

@@ -63,16 +63,24 @@
 }
 
 # counting config
-assert args.species in ["swallow", "bat"]
-CNT_CFG = {
-    "count_scaling":    1.2,    # the detector model predicts boxes that "trace roosts", enlarge to get a bounding box
-    "max_height":       5000,   # 5000m: this is and should be much higher than roosts' normal height (~2000m)
-    "rcs":              get_bird_rcs(54) if args.species == "swallow" else 4.519,
-    "threshold":        21630 if args.species == "swallow" else 21630891,
-                                # threshold above which we consider reflectivity to be too high in the linear scale:
-                                # 30dbZ -> 21630, 35dbZ -> 68402;
-                                # 60dbZ -> 21630891
-}
+if args.species == "swallow":  # Entire US deployment
+    CNT_CFG = {
+        "count_scaling":    1.2,    # the detector model predicts boxes that "trace roosts", enlarge to get a bounding box
+        "max_height":       5000,   # 5000m: this is and should be much higher than roosts' normal height (~2000m)
+        "rcs":              1,      # set to 1, multiply counts by rcs in the sweep-level stage of post-processing.
+                                    # once considered setting get_bird_rcs(54)
+        "threshold":        216309, # threshold above which we consider reflectivity to be too high in the linear scale:
+                                    # 30dbZ -> 21630, 35dbZ -> 68402, 40dbZ -> 216309, 60dbZ -> 21630891
+    }
+elif args.species == "bat":  # Texas bats deployment
+    CNT_CFG = {
+        "count_scaling":    1.2,
+        "max_height":       5000,
+        "rcs":              4.519,
+        "threshold":        21630891,  # 60dbZ
+    }
+else:
+    raise ValueError("args.species has to be either swallow or bat")
 
 # directories
 DIRS = {
@@ -88,7 +96,7 @@
 ######################### Run #########################
 roost_system = RoostSystem(args, DET_CFG, PP_CFG, CNT_CFG, DIRS)
 
-days = get_days_list(args.start, args.end) # timestamps that indicate the beginning of dates, no time zone info
+days = get_days_list(args.start, args.end)  # timestamps that indicate the beginning of dates, no time zone info
 print("Total number of days: %d" % len(days), flush=True)
 for day_idx, day in enumerate(days):
     process_start_time = time.time()

tools/demo.sh

@@ -22,21 +22,21 @@ python demo.py \
 --sun_activity ${SUN_ACTIVITY} --min_before ${MIN_BEFORE} --min_after ${MIN_AFTER} \
 --data_root ${OUTPUT_ROOT} --model_version ${MODEL_VERSION}
 
-# transfer outputs
-cd ${OUTPUT_ROOT}
-# (1) images to visualize dz05 and vr05
-ssh ${DST_HOST} mkdir -p ${DST_IMG}/${DATASET}
-rsync -avz ui/img/* ${DST_HOST}:${DST_IMG}/${DATASET}/
-# (2) csv for scans_and_tracks
-ssh ${DST_HOST} mkdir -p ${DST_PRED}/${DATASET}
-rsync -avz ui/scans_and_tracks/* ${DST_HOST}:${DST_PRED}/${DATASET}/
-# (3) arrays
-rsync -a arrays/ ${DST_HOST}:${DST_ARRAY}
-# (4) logs and empty scans directory
-ssh ${DST_HOST} mkdir -p ${DST_OTHERS}/${DATASET}/logs
-rsync -a logs/ ${DST_HOST}:${DST_OTHERS}/${DATASET}/logs/
-ssh ${DST_HOST} mkdir -p ${DST_OTHERS}/${DATASET}/scans
-rsync -a scans/ ${DST_HOST}:${DST_OTHERS}/${DATASET}/scans/
-# (5) slurm_logs
-ssh ${DST_HOST} mkdir -p ${DST_OTHERS}/slurm_logs
-scp -r ${SRC_SLURM}/${DATASET} ${DST_HOST}:${DST_OTHERS}/slurm_logs/
+# transfer outputs TODO: avoid repeated transfer, make it station-year specific
+#cd ${OUTPUT_ROOT}
+## (1) images to visualize dz05 and vr05
+#ssh ${DST_HOST} mkdir -p ${DST_IMG}/${DATASET}
+#rsync -avz ui/img/* ${DST_HOST}:${DST_IMG}/${DATASET}/
+## (2) csv for scans_and_tracks
+#ssh ${DST_HOST} mkdir -p ${DST_PRED}/${DATASET}
+#rsync -avz ui/scans_and_tracks/* ${DST_HOST}:${DST_PRED}/${DATASET}/
+## (3) arrays
+#rsync -a arrays/ ${DST_HOST}:${DST_ARRAY}
+## (4) logs and empty scans directory
+#ssh ${DST_HOST} mkdir -p ${DST_OTHERS}/${DATASET}/logs
+#rsync -a logs/ ${DST_HOST}:${DST_OTHERS}/${DATASET}/logs/
+#ssh ${DST_HOST} mkdir -p ${DST_OTHERS}/${DATASET}/scans
+#rsync -a scans/ ${DST_HOST}:${DST_OTHERS}/${DATASET}/scans/
+## (5) slurm_logs
+#ssh ${DST_HOST} mkdir -p ${DST_OTHERS}/slurm_logs
+#scp -r ${SRC_SLURM}/${DATASET} ${DST_HOST}:${DST_OTHERS}/slurm_logs/

tools/launch_demo.py

@@ -6,34 +6,28 @@
 
 # Config for deploying the system
 NUM_CPUS = 7
+
 # deployment station, start date (inclusive), end date (inclusive)
 # specify either
-# STATIONS = ["KABR", "KABX", "KAKQ"]
-# TIMES = [("20220101", "20221231"),]
-# STATIONS = ["KOKX"]
-# TIMES = [(f"{year}0601", f"{year}1031") for year in range(2000, 2023)]
+STATIONS = ["KOKX"]
+TIMES = [(f"{year}0101", f"{year}1231") for year in range(9999, 10000)]
 # or
-STATIONS_TIMES = [
-    ("KTYX", "20200805", "20200806"),
-    # ("KTYX", "20200101", "20201231"),
-    # ("KTYX", "20220101", "20221231"),
-    # ("KLIX", "20200101", "20201231"),
-    # ("KLIX", "20220101", "20221231"),
-    # ("KDAX", "20200101", "20201231"),
-    # ("KDAX", "20220101", "20221231"),
-    # ("KTLX", "20200101", "20201231"),
-    # ("KTLX", "20220101", "20221231"),
-]
+# STATIONS_TIMES = [
+#     ("KTYX", "20200805", "20200806"),
+# ]
+
 SPECIES = "swallow"
 SUN_ACTIVITY = "sunrise" # bird activities occur around sunrise
 MIN_BEFORE = 30
 MIN_AFTER = 90
+
 # directory for system outputs
 MODEL_VERSION = "v3"
-EXPERIMENT_NAME = f"us_sunrise_{MODEL_VERSION}_pilot0119" # dataset name
+EXPERIMENT_NAME = f"us_sunrise_{MODEL_VERSION}" # dataset name
 OUTPUT_ROOT = f"/mnt/nfs/scratch1/wenlongzhao/roosts_data/{EXPERIMENT_NAME}"
-SRC_SLURM = "~/work1/roost-system/tools/slurm_logs"
+
 # Config for transferring outputs from the computing cluster to our server
+SRC_SLURM = "~/work1/roost-system/tools/slurm_logs"
 DST_HOST = "doppler.cs.umass.edu"
 DST_IMG = "/var/www/html/roost/img" # dz05 and vr05 jpg images
 DST_PRED = "/scratch2/wenlongzhao/roostui/data" # csv for scans_and_tracks
@@ -63,8 +57,14 @@
 
     # Now we request cpus via slurm to run the job
     cmd = f'''sbatch \
-    --job-name="{station}{start}_{end}" --output="{slurm_output}" --error="{slurm_error}" \
-    --partition=longq --nodes=1 --ntasks=1 --cpus-per-task={NUM_CPUS} --mem-per-cpu=2000 \
+    --job-name="{station}{start}_{end}" \
+    --output="{slurm_output}" \
+    --error="{slurm_error}" \
+    --partition=longq \
+    --nodes=1 \
+    --ntasks=1 \
+    --cpus-per-task={NUM_CPUS} \
+    --mem-per-cpu=2000 \
     --time=7-00:00:00 \
     demo.sh \
     {SPECIES} {station} {start} {end} \

tools/launch_demo_bat.py

@@ -1,28 +1,36 @@
+"""
+This script can launch many jobs in parallel, each for a station-year and on separate cpus
+"""
 import os
 import time
 
+# Config for deploying the system
 NUM_CPUS = 7
+
 # deployment station, start date (inclusive), end date (inclusive)
-# specify either
-STATIONS = ["KEWX"] # ["KDFX", "KSJT", "KGRK"]
+STATIONS = ["KEWX", "KDFX", "KSJT", "KGRK"]
 TIMES = []
-for year in range(2000, 2021):
-    # for (start_date, end_date) in [("0101", "0331"), ("0401", "0630"), ("0701", "0930"), ("1001", "1231")]:
+for year in [1994, 1995, 1996, 1997, 1998, 1999, 2021, 2022, 2023]:
     for (start_date, end_date) in [("0101", "1231")]:
         TIMES.append((str(year)+start_date, str(year)+end_date))
-# or
-# STATIONS_TIMES = [
-#     ("KDFX", "20060101", "20061231"),
-# ]
 
 SPECIES = "bat"
 SUN_ACTIVITY = "sunset" # bat activities occur around sunset
-MIN_BEFORE = 150
+MIN_BEFORE = 90
 MIN_AFTER = 150
+
 # directory for system outputs
 MODEL_VERSION = "v3"
-EXPERIMENT_NAME = f"texas_bats_{MODEL_VERSION}_long"
-DATA_ROOT = f"/mnt/nfs/scratch1/wenlongzhao/roosts_data/{EXPERIMENT_NAME}"
+EXPERIMENT_NAME = f"texas_bats_{MODEL_VERSION}" # dataset name
+OUTPUT_ROOT = f"/mnt/nfs/scratch1/wenlongzhao/roosts_data/{EXPERIMENT_NAME}"
+
+# Config for transferring outputs from the computing cluster to our server
+SRC_SLURM = "~/work1/roost-system/tools/slurm_logs"
+DST_HOST = "doppler.cs.umass.edu"
+DST_IMG = "/var/www/html/roost/img" # dz05 and vr05 jpg images
+DST_PRED = "/scratch2/wenlongzhao/roostui/data" # csv for scans_and_tracks
+DST_ARRAY = "/scratch2/wenlongzhao/RadarNPZ/v0.3.0/" # arrays
+DST_OTHERS = "/scratch2/wenlongzhao/roosts_deployment_outputs" # logs, scans
 
 try:
     assert STATIONS_TIMES
@@ -43,20 +51,23 @@
     os.system(f"export OPENBLAS_NUM_THREADS={NUM_CPUS}")
     os.system(f"export OMP_NUM_THREADS={NUM_CPUS}")
 
+    # Now we request cpus via slurm to run the job
     cmd = f'''sbatch \
     --job-name="{station}{start}_{end}" \
     --output="{slurm_output}" \
     --error="{slurm_error}" \
+    --partition=longq \
     --nodes=1 \
     --ntasks=1 \
     --cpus-per-task={NUM_CPUS} \
     --mem-per-cpu=2000 \
-    --partition=longq \
     --time=7-00:00:00 \
-    demo.sbatch \
-    --species {SPECIES} --station {station} --start {start} --end {end} \
-    --sun_activity {SUN_ACTIVITY} --min_before {MIN_BEFORE} --min_after {MIN_AFTER} \
-    --data_root {DATA_ROOT} --model_version {MODEL_VERSION}'''
+    demo.sh \
+    {SPECIES} {station} {start} {end} \
+    {SUN_ACTIVITY} {MIN_BEFORE} {MIN_AFTER} \
+    {OUTPUT_ROOT} {MODEL_VERSION} \
+    {EXPERIMENT_NAME} {SRC_SLURM} \
+    {DST_HOST} {DST_IMG} {DST_PRED} {DST_ARRAY} {DST_OTHERS}'''
 
     os.system(cmd)
     time.sleep(1)