Miscellaneous config changes.

- Use 32 workers when materializing vessel detection crops.
- Just use Jupiter for Sentinel-2 job launcher.
- Remove redundant code in Sentinel-2 vessel detection pipeline that could hide an error.
- Ecosystem mapping: update to use latest dataset version
- Minderoo: removed unneeded inference config and update training config to new config format

data/maldives_ecosystem_mapping/config_sentinel2.yaml

@@ -30,7 +30,7 @@ model:
 data:
   class_path: rslearn.train.data_module.RslearnDataModule
   init_args:
-    path: gcs://rslearn-eai/datasets/maldives_ecosystem_mapping/dataset_v1/20240924/
+    path: gcs://rslearn-eai/datasets/maldives_ecosystem_mapping/dataset_v1/20241017/
     inputs:
       sentinel2:
         data_type: "raster"
@@ -146,7 +146,7 @@ trainer:
         logging_interval: "epoch"
     - class_path: rslearn.train.prediction_writer.RslearnWriter
       init_args:
-        path: gcs://rslearn-eai/datasets/maldives_ecosystem_mapping/dataset_v1/20240924/
+        path: gcs://rslearn-eai/datasets/maldives_ecosystem_mapping/dataset_v1/20241017/
         output_layer: output
         selector: ["segment"]
     - class_path: lightning.pytorch.callbacks.ModelCheckpoint

one_off_projects/minderoo/classify/config.yaml

@@ -17,14 +17,15 @@ model:
                 num_fc_layers: 1
             - class_path: rslearn.train.tasks.classification.ClassificationHead
     lr: 0.0001
-    plateau_factor: 0.1
-    plateau_patience: 10
+    plateau: true
+    plateau_factor: 0.5
+    plateau_patience: 2
     plateau_min_lr: 0
-    plateau_cooldown: 0
+    plateau_cooldown: 5
 data:
   class_path: rslearn.train.data_module.RslearnDataModule
   init_args:
-    root_dir: /data/favyenb/dvim/rslearn_classify/
+    path: /data/favyenb/dvim/rslearn_classify/
     inputs:
       image:
         data_type: "raster"
@@ -88,8 +89,14 @@ trainer:
         logging_interval: "epoch"
     - class_path: rslearn.train.prediction_writer.RslearnWriter
       init_args:
-        root_dir: /data/favyenb/dvim/rslearn_classify/
+        path: /data/favyenb/dvim/rslearn_classify/
         output_layer: output
         selector: ["class"]
+    - class_path: lightning.pytorch.callbacks.ModelCheckpoint
+      init_args:
+        save_top_k: 1
+        save_last: true
+        monitor: val_class/accuracy
+        mode: max
 rslp_project: minderoo_classify
-rslp_experiment: data_20240811_satlaspretrain_flip_256_00
+rslp_experiment: data_20241028_satlaspretrain_flip_256_00

rslp/landsat_vessels/predict_pipeline.py

@@ -215,7 +215,7 @@ def run_classifier(
             window.save_layer_datas(dict(LANDSAT_LAYER_NAME=layer_data))
 
     logger.info("materialize dataset")
-    apply_windows_args = ApplyWindowsArgs(group=group)
+    apply_windows_args = ApplyWindowsArgs(group=group, workers=32)
     materialize_pipeline_args = MaterializePipelineArgs(
         disabled_layers=[],
         prepare_args=PrepareArgs(apply_windows_args=apply_windows_args),

rslp/sentinel2_vessels/job_launcher.py

@@ -61,10 +61,7 @@ def launch_job(image_name: str, tasks: list[PredictionTask]) -> None:
             ],
             constraints=Constraints(
                 cluster=[
-                    # Don't use Renton cirrascale since it loads very slowly from Weka.
-                    "ai2/prior-elanding",
                     "ai2/jupiter-cirrascale-2",
-                    "ai2/neptune-cirrascale",
                 ]
             ),
             preemptible=True,

rslp/sentinel2_vessels/predict_pipeline.py

@@ -137,7 +137,7 @@ def get_vessel_detections(
         window.save_layer_datas(dict(SENTINEL2_LAYER_NAME=layer_data))
 
     logger.info("Materialize dataset for Sentinel-2 Vessel Detection")
-    apply_windows_args = ApplyWindowsArgs(group=group, workers=1)
+    apply_windows_args = ApplyWindowsArgs(group=group, workers=32)
     materialize_pipeline_args = MaterializePipelineArgs(
         disabled_layers=[],
         prepare_args=PrepareArgs(apply_windows_args=apply_windows_args),
@@ -297,9 +297,6 @@ def predict_pipeline(tasks: list[PredictionTask], scratch_path: str) -> None:
         with crop_fname.open("wb") as f:
             Image.fromarray(image.transpose(1, 2, 0)).save(f, format="PNG")
 
-        if scene_id not in json_vessels_by_scene:
-            json_vessels_by_scene[scene_id] = []
-
         json_vessels_by_scene[scene_id].append(
             dict(
                 longitude=lon,