experiments

chris-santiago · Mar 14, 2024 · 08d8f31 · 08d8f31
1 parent a746c97
commit 08d8f31
Show file tree

Hide file tree

Showing 65 changed files with 4,488 additions and 20,772 deletions.
diff --git a/data/train-s4e2-ps.csv b/data/train-s4e2-ps.csv
diff --git a/data/valid-s4e2-ps.csv b/data/valid-s4e2-ps.csv
diff --git a/outputs/eval/2024-03-13/09-39-41/.hydra/config.yaml b/outputs/eval/2024-03-13/09-39-41/.hydra/config.yaml
@@ -0,0 +1,77 @@
+data:
+  batch_size: 128
+  n_workers: 10
+  name: obesity
+  train:
+    _target_: torch.utils.data.DataLoader
+    dataset:
+      _target_: scarf.data.CSVDataset
+      file_path: data/train.csv
+    batch_size: ${data.batch_size}
+    shuffle: true
+    num_workers: ${data.n_workers}
+  valid:
+    _target_: torch.utils.data.DataLoader
+    dataset:
+      _target_: scarf.data.CSVDataset
+      file_path: data/valid.csv
+    batch_size: ${data.batch_size}
+    shuffle: false
+    num_workers: ${data.n_workers}
+  test:
+    _target_: torch.utils.data.DataLoader
+    dataset:
+      _target_: scarf.data.CSVDataset
+      file_path: data/test.csv
+    batch_size: ${data.batch_size}
+    shuffle: false
+    num_workers: ${data.n_workers}
+seed: 43
+models:
+  scarf:
+    name: SCARF
+    estimator:
+      _target_: scarf.models.scarf.SCARFLearner.load_from_checkpoint
+      checkpoint_path: outputs/scarf-learner/train/2024-03-13/09-33-41/checkpoints/epoch=14-step=210.ckpt
+      map_location:
+        _target_: torch.device
+        _args_:
+        - cpu
+  gbm:
+    name: GBM
+    estimator:
+      _target_: joblib.load
+      filename: outputs/gradient-booster/train_baseline/2024-03-06/15-28-28/gradient-booster-estimator.joblib
+  linear:
+    name: Linear
+    estimator:
+      _target_: joblib.load
+      filename: outputs/linear/train_baseline/2024-03-06/15-29-56/linear-estimator.joblib
+task: multiclass
+num_classes: 7
+metrics:
+  Accuracy:
+    _target_: torchmetrics.Accuracy
+    task: ${task}
+    num_classes: ${num_classes}
+  AUROC:
+    _target_: torchmetrics.AUROC
+    task: ${task}
+    num_classes: ${num_classes}
+  AUPRC:
+    _target_: torchmetrics.AveragePrecision
+    task: ${task}
+    num_classes: ${num_classes}
+  F1-Score:
+    _target_: torchmetrics.F1Score
+    task: ${task}
+    num_classes: ${num_classes}
+plots:
+  Confusion Matrix:
+    _target_: torchmetrics.ConfusionMatrix
+    task: ${task}
+    num_classes: ${num_classes}
+  Receiver Operator Curve:
+    _target_: torchmetrics.ROC
+    task: ${task}
+    num_classes: ${num_classes}
diff --git a/outputs/eval/2024-03-13/09-39-41/.hydra/hydra.yaml b/outputs/eval/2024-03-13/09-39-41/.hydra/hydra.yaml
@@ -0,0 +1,167 @@
+hydra:
+  run:
+    dir: outputs/${hydra.job.name}/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: outputs/${hydra.job.name}/multirun
+    subdir: ${hydra.job.override_dirname}/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  launcher:
+    _target_: hydra_plugins.hydra_joblib_launcher.joblib_launcher.JoblibLauncher
+    n_jobs: -1
+    backend: null
+    prefer: processes
+    require: null
+    verbose: 0
+    timeout: null
+    pre_dispatch: 2*n_jobs
+    batch_size: auto
+    temp_folder: null
+    max_nbytes: null
+    mmap_mode: r
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task: []
+  job:
+    name: eval
+    chdir: null
+    override_dirname: ''
+    id: ???
+    num: ???
+    config_name: eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /Users/chrissantiago/Dropbox/GitHub/scarf
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /Users/chrissantiago/Dropbox/GitHub/scarf/scarf/conf
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /Users/chrissantiago/Dropbox/GitHub/scarf/outputs/eval/2024-03-13/09-39-41
+    choices:
+      data: learner
+      hydra: eval
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: joblib
+      hydra/output: default
+  verbose: false
diff --git a/outputs/eval/2024-03-13/09-39-41/.hydra/overrides.yaml b/outputs/eval/2024-03-13/09-39-41/.hydra/overrides.yaml
@@ -0,0 +1 @@
+[]
diff --git a/outputs/eval/2024-03-13/09-39-41/Confusion Matrix - GBM.png b/outputs/eval/2024-03-13/09-39-41/Confusion Matrix - GBM.png
diff --git a/outputs/eval/2024-03-13/09-39-41/Confusion Matrix - Linear.png b/outputs/eval/2024-03-13/09-39-41/Confusion Matrix - Linear.png
diff --git a/outputs/eval/2024-03-13/09-39-41/Confusion Matrix - SCARF.png b/outputs/eval/2024-03-13/09-39-41/Confusion Matrix - SCARF.png
diff --git a/outputs/eval/2024-03-13/09-39-41/Receiver Operator Curve - GBM.png b/outputs/eval/2024-03-13/09-39-41/Receiver Operator Curve - GBM.png
diff --git a/outputs/eval/2024-03-13/09-39-41/Receiver Operator Curve - Linear.png b/outputs/eval/2024-03-13/09-39-41/Receiver Operator Curve - Linear.png
diff --git a/outputs/eval/2024-03-13/09-39-41/Receiver Operator Curve - SCARF.png b/outputs/eval/2024-03-13/09-39-41/Receiver Operator Curve - SCARF.png
diff --git a/outputs/eval/2024-03-13/09-39-41/results.json b/outputs/eval/2024-03-13/09-39-41/results.json
@@ -0,0 +1,20 @@
+{
+    "scarf": {
+        "Accuracy": 0.9292,
+        "AUROC": 0.9941,
+        "AUPRC": 0.9681,
+        "F1-Score": 0.9292
+    },
+    "gbm": {
+        "Accuracy": 0.9575,
+        "AUROC": 0.9964,
+        "AUPRC": 0.9753,
+        "F1-Score": 0.9575
+    },
+    "linear": {
+        "Accuracy": 0.9764,
+        "AUROC": 0.9983,
+        "AUPRC": 0.9904,
+        "F1-Score": 0.9764
+    }
+}
diff --git a/outputs/eval/2024-03-13/09-46-11/.hydra/config.yaml b/outputs/eval/2024-03-13/09-46-11/.hydra/config.yaml
@@ -0,0 +1,77 @@
+data:
+  batch_size: 128
+  n_workers: 10
+  name: obesity
+  train:
+    _target_: torch.utils.data.DataLoader
+    dataset:
+      _target_: scarf.data.CSVDataset
+      file_path: data/train.csv
+    batch_size: ${data.batch_size}
+    shuffle: true
+    num_workers: ${data.n_workers}
+  valid:
+    _target_: torch.utils.data.DataLoader
+    dataset:
+      _target_: scarf.data.CSVDataset
+      file_path: data/valid.csv
+    batch_size: ${data.batch_size}
+    shuffle: false
+    num_workers: ${data.n_workers}
+  test:
+    _target_: torch.utils.data.DataLoader
+    dataset:
+      _target_: scarf.data.CSVDataset
+      file_path: data/test.csv
+    batch_size: ${data.batch_size}
+    shuffle: false
+    num_workers: ${data.n_workers}
+seed: 43
+models:
+  scarf:
+    name: SCARF
+    estimator:
+      _target_: scarf.models.scarf.SCARFLearner.load_from_checkpoint
+      checkpoint_path: outputs/scarf-learner/train/2024-03-13/09-41-32/checkpoints/epoch=3-step=56.ckpt
+      map_location:
+        _target_: torch.device
+        _args_:
+        - cpu
+  gbm:
+    name: GBM
+    estimator:
+      _target_: joblib.load
+      filename: outputs/gradient-booster/train_baseline/2024-03-06/15-28-28/gradient-booster-estimator.joblib
+  linear:
+    name: Linear
+    estimator:
+      _target_: joblib.load
+      filename: outputs/linear/train_baseline/2024-03-06/15-29-56/linear-estimator.joblib
+task: multiclass
+num_classes: 7
+metrics:
+  Accuracy:
+    _target_: torchmetrics.Accuracy
+    task: ${task}
+    num_classes: ${num_classes}
+  AUROC:
+    _target_: torchmetrics.AUROC
+    task: ${task}
+    num_classes: ${num_classes}
+  AUPRC:
+    _target_: torchmetrics.AveragePrecision
+    task: ${task}
+    num_classes: ${num_classes}
+  F1-Score:
+    _target_: torchmetrics.F1Score
+    task: ${task}
+    num_classes: ${num_classes}
+plots:
+  Confusion Matrix:
+    _target_: torchmetrics.ConfusionMatrix
+    task: ${task}
+    num_classes: ${num_classes}
+  Receiver Operator Curve:
+    _target_: torchmetrics.ROC
+    task: ${task}
+    num_classes: ${num_classes}