From c1e7c32bdc2ce6bbabbb31b96cc95e19dadf14f5 Mon Sep 17 00:00:00 2001 From: Samet Akcay Date: Tue, 5 Apr 2022 07:15:34 -0700 Subject: [PATCH 1/7] remove `save_to_csv` flag from config __init__ --- anomalib/utils/callbacks/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/anomalib/utils/callbacks/__init__.py b/anomalib/utils/callbacks/__init__.py index ad4e120585..3edaa8302f 100644 --- a/anomalib/utils/callbacks/__init__.py +++ b/anomalib/utils/callbacks/__init__.py @@ -107,8 +107,4 @@ def get_callbacks(config: Union[ListConfig, DictConfig]) -> List[Callback]: ) ) - if "save_to_csv" in config.project.keys(): - if config.project.save_to_csv: - callbacks.append(SaveToCSVCallback()) - return callbacks From 6004493e0348caa801f164e50d3ed5fdc02863db Mon Sep 17 00:00:00 2001 From: Samet Akcay Date: Tue, 5 Apr 2022 07:17:53 -0700 Subject: [PATCH 2/7] Removed `save_to_csv` from config files --- anomalib/models/cflow/config.yaml | 1 - anomalib/models/dfkde/config.yaml | 1 - anomalib/models/dfm/config.yaml | 1 - anomalib/models/ganomaly/config.yaml | 1 - anomalib/models/padim/config.yaml | 1 - anomalib/models/patchcore/config.yaml | 1 - anomalib/models/stfpm/config.yaml | 1 - 7 files changed, 7 deletions(-) diff --git a/anomalib/models/cflow/config.yaml b/anomalib/models/cflow/config.yaml index f321915493..ac8d447f1a 100644 --- a/anomalib/models/cflow/config.yaml +++ b/anomalib/models/cflow/config.yaml @@ -40,7 +40,6 @@ project: path: ./results log_images_to: [local] logger: false - save_to_csv: false # PL Trainer Args. Don't add extra parameter here. trainer: diff --git a/anomalib/models/dfkde/config.yaml b/anomalib/models/dfkde/config.yaml index c1fb4982ad..8ca6587376 100644 --- a/anomalib/models/dfkde/config.yaml +++ b/anomalib/models/dfkde/config.yaml @@ -28,7 +28,6 @@ project: path: ./results log_images_to: [] logger: false - save_to_csv: false # PL Trainer Args. Don't add extra parameter here. trainer: diff --git a/anomalib/models/dfm/config.yaml b/anomalib/models/dfm/config.yaml index b8d8faa626..725f67d36a 100755 --- a/anomalib/models/dfm/config.yaml +++ b/anomalib/models/dfm/config.yaml @@ -27,7 +27,6 @@ project: path: ./results log_images_to: [] logger: false - save_to_csv: false # PL Trainer Args. Don't add extra parameter here. trainer: diff --git a/anomalib/models/ganomaly/config.yaml b/anomalib/models/ganomaly/config.yaml index 8ef71c4900..b9e9684e52 100644 --- a/anomalib/models/ganomaly/config.yaml +++ b/anomalib/models/ganomaly/config.yaml @@ -44,7 +44,6 @@ project: path: ./results log_images_to: [] logger: false - save_to_csv: false optimization: openvino: diff --git a/anomalib/models/padim/config.yaml b/anomalib/models/padim/config.yaml index 859a3ad23e..dd8d1b851f 100644 --- a/anomalib/models/padim/config.yaml +++ b/anomalib/models/padim/config.yaml @@ -37,7 +37,6 @@ project: path: ./results log_images_to: ["local"] logger: false - save_to_csv: false optimization: openvino: diff --git a/anomalib/models/patchcore/config.yaml b/anomalib/models/patchcore/config.yaml index 75b910da2a..69525ffa80 100644 --- a/anomalib/models/patchcore/config.yaml +++ b/anomalib/models/patchcore/config.yaml @@ -39,7 +39,6 @@ project: path: ./results log_images_to: [local] logger: false - save_to_csv: false # PL Trainer Args. Don't add extra parameter here. trainer: diff --git a/anomalib/models/stfpm/config.yaml b/anomalib/models/stfpm/config.yaml index e605dbdaef..fd19a60835 100644 --- a/anomalib/models/stfpm/config.yaml +++ b/anomalib/models/stfpm/config.yaml @@ -44,7 +44,6 @@ project: path: ./results log_images_to: [local] logger: false - save_to_csv: false optimization: openvino: From a771104a4e3045ed08a16586e9478e287be30638 Mon Sep 17 00:00:00 2001 From: Samet Akcay Date: Tue, 5 Apr 2022 07:27:21 -0700 Subject: [PATCH 3/7] Added csv option to logger in the config files --- anomalib/models/cflow/config.yaml | 2 +- anomalib/models/dfkde/config.yaml | 2 +- anomalib/models/dfm/config.yaml | 2 +- anomalib/models/ganomaly/config.yaml | 2 +- anomalib/models/padim/config.yaml | 2 +- anomalib/models/patchcore/config.yaml | 2 +- anomalib/models/stfpm/config.yaml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/anomalib/models/cflow/config.yaml b/anomalib/models/cflow/config.yaml index ac8d447f1a..a5adb95804 100644 --- a/anomalib/models/cflow/config.yaml +++ b/anomalib/models/cflow/config.yaml @@ -39,7 +39,7 @@ project: seed: 0 path: ./results log_images_to: [local] - logger: false + logger: false # options: [tensorboard, wandb, csv] or combinations. # PL Trainer Args. Don't add extra parameter here. trainer: diff --git a/anomalib/models/dfkde/config.yaml b/anomalib/models/dfkde/config.yaml index 8ca6587376..6195153c9f 100644 --- a/anomalib/models/dfkde/config.yaml +++ b/anomalib/models/dfkde/config.yaml @@ -27,7 +27,7 @@ project: seed: 42 path: ./results log_images_to: [] - logger: false + logger: false # options: [tensorboard, wandb, csv] or combinations. # PL Trainer Args. Don't add extra parameter here. trainer: diff --git a/anomalib/models/dfm/config.yaml b/anomalib/models/dfm/config.yaml index 725f67d36a..6b08d11668 100755 --- a/anomalib/models/dfm/config.yaml +++ b/anomalib/models/dfm/config.yaml @@ -26,7 +26,7 @@ project: seed: 42 path: ./results log_images_to: [] - logger: false + logger: false # options: [tensorboard, wandb, csv] or combinations. # PL Trainer Args. Don't add extra parameter here. trainer: diff --git a/anomalib/models/ganomaly/config.yaml b/anomalib/models/ganomaly/config.yaml index b9e9684e52..2e9c351f6b 100644 --- a/anomalib/models/ganomaly/config.yaml +++ b/anomalib/models/ganomaly/config.yaml @@ -43,7 +43,7 @@ project: seed: 0 path: ./results log_images_to: [] - logger: false + logger: false # options: [tensorboard, wandb, csv] or combinations. optimization: openvino: diff --git a/anomalib/models/padim/config.yaml b/anomalib/models/padim/config.yaml index dd8d1b851f..18317b5bc5 100644 --- a/anomalib/models/padim/config.yaml +++ b/anomalib/models/padim/config.yaml @@ -36,7 +36,7 @@ project: seed: 42 path: ./results log_images_to: ["local"] - logger: false + logger: false # options: [tensorboard, wandb, csv] or combinations. optimization: openvino: diff --git a/anomalib/models/patchcore/config.yaml b/anomalib/models/patchcore/config.yaml index 69525ffa80..2a4a01a47b 100644 --- a/anomalib/models/patchcore/config.yaml +++ b/anomalib/models/patchcore/config.yaml @@ -38,7 +38,7 @@ project: seed: 0 path: ./results log_images_to: [local] - logger: false + logger: false # options: [tensorboard, wandb, csv] or combinations. # PL Trainer Args. Don't add extra parameter here. trainer: diff --git a/anomalib/models/stfpm/config.yaml b/anomalib/models/stfpm/config.yaml index fd19a60835..3831a5e1bd 100644 --- a/anomalib/models/stfpm/config.yaml +++ b/anomalib/models/stfpm/config.yaml @@ -43,7 +43,7 @@ project: seed: 0 path: ./results log_images_to: [local] - logger: false + logger: false # options: [tensorboard, wandb, csv] or combinations. optimization: openvino: From ea5d9053451ffa78c400ca9d888f05f0f67e1561 Mon Sep 17 00:00:00 2001 From: Samet Akcay Date: Tue, 5 Apr 2022 07:28:27 -0700 Subject: [PATCH 4/7] Add csv tests to logger tests --- .../pre_merge/utils/loggers/test_get_logger.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/pre_merge/utils/loggers/test_get_logger.py b/tests/pre_merge/utils/loggers/test_get_logger.py index e48798d7a2..ec7dd60ac3 100644 --- a/tests/pre_merge/utils/loggers/test_get_logger.py +++ b/tests/pre_merge/utils/loggers/test_get_logger.py @@ -16,6 +16,7 @@ import pytest from omegaconf import OmegaConf +from pytorch_lightning.loggers import CSVLogger from anomalib.utils.loggers import ( AnomalibTensorBoardLogger, @@ -46,12 +47,24 @@ def test_get_logger(): # get tensorboard config.project.logger = "tensorboard" logger = get_logger(config=config) - assert isinstance(logger, AnomalibTensorBoardLogger) + assert isinstance(logger[0], AnomalibTensorBoardLogger) # get wandb logger config.project.logger = "wandb" logger = get_logger(config=config) - assert isinstance(logger, AnomalibWandbLogger) + assert isinstance(logger[0], AnomalibWandbLogger) + + # get csv logger. + config.project.logger = "csv" + logger = get_logger(config=config) + assert isinstance(logger[0], CSVLogger) + + # get multiple loggers + config.project.logger = ["tensorboard", "wandb", "csv"] + logger = get_logger(config=config) + assert isinstance(logger[0], AnomalibTensorBoardLogger) + assert isinstance(logger[1], AnomalibWandbLogger) + assert isinstance(logger[2], CSVLogger) # raise unknown with pytest.raises(UnknownLogger): From 68e17e45ae82782b8009f5d366de716a61d2369f Mon Sep 17 00:00:00 2001 From: Samet Akcay Date: Tue, 5 Apr 2022 07:30:50 -0700 Subject: [PATCH 5/7] remove save to csv callback --- anomalib/utils/callbacks/__init__.py | 2 - anomalib/utils/callbacks/save_to_csv.py | 51 ------------------------- 2 files changed, 53 deletions(-) delete mode 100644 anomalib/utils/callbacks/save_to_csv.py diff --git a/anomalib/utils/callbacks/__init__.py b/anomalib/utils/callbacks/__init__.py index 3edaa8302f..13bda162c8 100644 --- a/anomalib/utils/callbacks/__init__.py +++ b/anomalib/utils/callbacks/__init__.py @@ -25,7 +25,6 @@ from .cdf_normalization import CdfNormalizationCallback from .min_max_normalization import MinMaxNormalizationCallback from .model_loader import LoadModelCallback -from .save_to_csv import SaveToCSVCallback from .timer import TimerCallback from .visualizer_callback import VisualizerCallback @@ -33,7 +32,6 @@ "LoadModelCallback", "TimerCallback", "VisualizerCallback", - "SaveToCSVCallback", ] diff --git a/anomalib/utils/callbacks/save_to_csv.py b/anomalib/utils/callbacks/save_to_csv.py deleted file mode 100644 index 0fc2f26034..0000000000 --- a/anomalib/utils/callbacks/save_to_csv.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Callback to save metrics to CSV.""" - -# Copyright (C) 2020 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions -# and limitations under the License. - -from pathlib import Path - -import numpy as np -import pandas as pd -from pytorch_lightning import Callback, Trainer - -from anomalib.models.components import AnomalyModule - - -class SaveToCSVCallback(Callback): - """Callback that saves the inference results of a model. - - The callback generates a csv file that saves the predicted label, the true label and the image name. - """ - - def __init__(self): - """Callback to save metrics to CSV.""" - - def on_test_epoch_end(self, _trainer: Trainer, pl_module: AnomalyModule) -> None: - """Save Results at the end of testing. - - Args: - _trainer (Trainer): Pytorch lightning trainer object (unused) - pl_module (LightningModule): Lightning modules derived from BaseAnomalyLightning object. - """ - results = pl_module.results - data_frame = pd.DataFrame( - { - "name": results.filenames, - "true_label": results.true_labels, - "pred_label": results.pred_labels.astype(int), - "wrong_prediction": np.logical_xor(results.true_labels, results.pred_labels).astype(int), - } - ) - data_frame.to_csv(Path(pl_module.hparams.project.path) / "results.csv") From 46370681f71e32ba56af916a83520c67b93c942f Mon Sep 17 00:00:00 2001 From: Samet Akcay Date: Tue, 5 Apr 2022 07:33:39 -0700 Subject: [PATCH 6/7] =?UTF-8?q?=F0=9F=9B=A0=20Modify=20`get=5Floggers`=20f?= =?UTF-8?q?unction=20to=20also=20get=20csv=20logger.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- anomalib/utils/loggers/__init__.py | 75 +++++++++++++++++------------- 1 file changed, 43 insertions(+), 32 deletions(-) diff --git a/anomalib/utils/loggers/__init__.py b/anomalib/utils/loggers/__init__.py index 63a8551bee..9afb3fd519 100644 --- a/anomalib/utils/loggers/__init__.py +++ b/anomalib/utils/loggers/__init__.py @@ -16,24 +16,26 @@ import os -from typing import Union +from typing import Iterable, List, Union from omegaconf.dictconfig import DictConfig from omegaconf.listconfig import ListConfig -from pytorch_lightning.loggers.base import LightningLoggerBase +from pytorch_lightning.loggers import CSVLogger, LightningLoggerBase from .tensorboard import AnomalibTensorBoardLogger from .wandb import AnomalibWandbLogger __all__ = ["AnomalibTensorBoardLogger", "get_logger", "AnomalibWandbLogger"] -AVAILABLE_LOGGERS = ["tensorboard", "wandb"] +AVAILABLE_LOGGERS = ["tensorboard", "wandb", "csv"] class UnknownLogger(Exception): """This is raised when the logger option in `config.yaml` file is set incorrectly.""" -def get_logger(config: Union[DictConfig, ListConfig]) -> Union[LightningLoggerBase, bool]: +def get_logger( + config: Union[DictConfig, ListConfig] +) -> Union[LightningLoggerBase, Iterable[LightningLoggerBase], bool]: """Return a logger based on the choice of logger in the config file. Args: @@ -45,32 +47,41 @@ def get_logger(config: Union[DictConfig, ListConfig]) -> Union[LightningLoggerBa Returns: Union[LightningLoggerBase, Iterable[LightningLoggerBase], bool]: Logger """ - logger: Union[LightningLoggerBase, bool] - if config.project.logger in [None, False]: - logger = False - - elif config.project.logger == "tensorboard": - logger = AnomalibTensorBoardLogger( - name="Tensorboard Logs", - save_dir=os.path.join(config.project.path, "logs"), - ) - - elif config.project.logger == "wandb": - wandb_logdir = os.path.join(config.project.path, "logs") - os.makedirs(wandb_logdir, exist_ok=True) - logger = AnomalibWandbLogger( - project=config.dataset.name, - name=f"{config.dataset.category} {config.model.name}", - save_dir=wandb_logdir, - ) - - else: - raise UnknownLogger( - f"Unknown logger type: {config.project.logger}. " - f"Available loggers are: {AVAILABLE_LOGGERS}.\n" - f"To enable the logger, set `project.logger` to `true` or use one of available loggers in config.yaml\n" - f"To disable the logger, set `project.logger` to `false`." - ) - - return logger + return False + + logger_list: List[LightningLoggerBase] = [] + if isinstance(config.project.logger, str): + config.project.logger = [config.project.logger] + + for logger in config.project.logger: + if logger == "tensorboard": + logger_list.append( + AnomalibTensorBoardLogger( + name="Tensorboard Logs", + save_dir=os.path.join(config.project.path, "logs"), + ) + ) + + elif logger == "wandb": + wandb_logdir = os.path.join(config.project.path, "logs") + os.makedirs(wandb_logdir, exist_ok=True) + logger_list.append( + AnomalibWandbLogger( + project=config.dataset.name, + name=f"{config.dataset.category} {config.model.name}", + save_dir=wandb_logdir, + ) + ) + elif logger == "csv": + logger_list.append(CSVLogger(save_dir=os.path.join(config.project.path, "logs"))) + + else: + raise UnknownLogger( + f"Unknown logger type: {config.project.logger}. " + f"Available loggers are: {AVAILABLE_LOGGERS}.\n" + f"To enable the logger, set `project.logger` to `true` or use one of available loggers in config.yaml\n" + f"To disable the logger, set `project.logger` to `false`." + ) + + return logger_list From ba6e462e31b30bc56192294f117537b6a60a0a09 Mon Sep 17 00:00:00 2001 From: Samet Akcay Date: Tue, 5 Apr 2022 08:49:55 -0700 Subject: [PATCH 7/7] =?UTF-8?q?=E2=9C=8F=EF=B8=8F=20Remove=20blank=20line.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- anomalib/utils/loggers/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/anomalib/utils/loggers/__init__.py b/anomalib/utils/loggers/__init__.py index 9afb3fd519..adbd0763b9 100644 --- a/anomalib/utils/loggers/__init__.py +++ b/anomalib/utils/loggers/__init__.py @@ -62,7 +62,6 @@ def get_logger( save_dir=os.path.join(config.project.path, "logs"), ) ) - elif logger == "wandb": wandb_logdir = os.path.join(config.project.path, "logs") os.makedirs(wandb_logdir, exist_ok=True) @@ -75,7 +74,6 @@ def get_logger( ) elif logger == "csv": logger_list.append(CSVLogger(save_dir=os.path.join(config.project.path, "logs"))) - else: raise UnknownLogger( f"Unknown logger type: {config.project.logger}. "