Skip to content

Commit

Permalink
Merge branch 'release/1.2-dev' into refactor/legacy-accel-plug
Browse files Browse the repository at this point in the history
  • Loading branch information
@mergify
mergify[bot] authored Jan 26, 2021
2 parents 5d46913 + c3587d3 commit 062f090
Show file tree
Hide file tree
Showing 12 changed files with 43 additions and 486 deletions.
226 changes: 0 additions & 226 deletions pytorch_lightning/core/step_result.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -700,232 +700,6 @@ def collate_tensors(items: Union[List, Tuple]) -> Union[Tensor, List, Tuple]:
return items


class EvalResult(Result):
def __init__(
self,
early_stop_on: Optional[Tensor] = None,
checkpoint_on: Optional[Tensor] = None,
hiddens: Optional[Tensor] = None,
):
"""
Used in val/train loop to auto-log to a logger or progress bar without needing to define
a _step_end or _epoch_end method
Example::
def validation_step(self, batch, batch_idx):
loss = ...
result = EvalResult()
result.log('val_loss', loss)
return result
def test_step(self, batch, batch_idx):
loss = ...
result = EvalResult()
result.log('val_loss', loss)
return result
Args:
early_stop_on: Metric to early stop on.
Should be a one element tensor if combined with default
:class:`~pytorch_lightning.callbacks.early_stopping.EarlyStopping`.
If this result is returned by
:meth:`~pytorch_lightning.core.lightning.LightningModule.validation_step`,
the specified value will be averaged across all steps.
checkpoint_on: Metric to checkpoint on.
Should be a one element tensor if combined with default checkpoint callback.
If this result is returned by
:meth:`~pytorch_lightning.core.lightning.LightningModule.validation_step`,
the specified value will be averaged across all steps.
hiddens:
"""

super().__init__(None, early_stop_on, checkpoint_on, hiddens)

def log(
self,
name,
value,
prog_bar: bool = False,
logger: bool = True,
on_step: bool = False,
on_epoch: bool = True,
reduce_fx: Callable = torch.mean,
tbptt_reduce_fx: Callable = torch.mean,
tbptt_pad_token: int = 0,
enable_graph: bool = False,
sync_dist: bool = False,
sync_dist_op: Union[Any, str] = 'mean',
sync_dist_group: Optional[Any] = None,
):
"""
Log a key, value
Example::
result.log('val_loss', loss)
# defaults used
result.log(
name,
value,
on_step=False,
on_epoch=True,
logger=True,
prog_bar=False,
reduce_fx=torch.mean
)
Args:
name: key name
value: value name
prog_bar: if True logs to the progress base
logger: if True logs to the logger
on_step: if True logs the output of validation_step or test_step
on_epoch: if True, logs the output of the training loop aggregated
reduce_fx: Torch.mean by default
tbptt_reduce_fx: function to reduce on truncated back prop
tbptt_pad_token: token to use for padding
enable_graph: if True, will not auto detach the graph
sync_dist: if True, reduces the metric across GPUs/TPUs
sync_dist_op: the op to sync across
sync_dist_group: the ddp group
"""
super().log(
name=name,
value=value,
prog_bar=prog_bar,
logger=logger,
on_step=on_step,
on_epoch=on_epoch,
reduce_fx=reduce_fx,
enable_graph=enable_graph,
sync_dist=sync_dist,
sync_dist_group=sync_dist_group,
sync_dist_op=sync_dist_op,
tbptt_pad_token=tbptt_pad_token,
tbptt_reduce_fx=tbptt_reduce_fx,
)

def log_dict(
self,
dictionary: dict,
prog_bar: bool = False,
logger: bool = True,
on_step: bool = False,
on_epoch: bool = True,
reduce_fx: Callable = torch.mean,
tbptt_reduce_fx: Callable = torch.mean,
tbptt_pad_token: int = 0,
enable_graph: bool = False,
sync_dist: bool = False,
sync_dist_op: Union[Any, str] = 'mean',
sync_dist_group: Optional[Any] = None,
):
"""
Log a dictonary of values at once
Example::
values = {'loss': loss, 'acc': acc, ..., 'metric_n': metric_n}
result.log_dict(values)
Args:
dictionary: key value pairs (str, tensors)
prog_bar: if True logs to the progress base
logger: if True logs to the logger
on_step: if True logs the output of validation_step or test_step
on_epoch: if True, logs the output of the training loop aggregated
reduce_fx: Torch.mean by default
tbptt_reduce_fx: function to reduce on truncated back prop
tbptt_pad_token: token to use for padding
enable_graph: if True, will not auto detach the graph
sync_dist: if True, reduces the metric across GPUs/TPUs
sync_dist_op: the op to sync across
sync_dist_group: the ddp group
"""
for k, v in dictionary.items():
self.log(
name=k,
value=v,
prog_bar=prog_bar,
logger=logger,
on_step=on_step,
on_epoch=on_epoch,
reduce_fx=reduce_fx,
enable_graph=enable_graph,
sync_dist=sync_dist,
sync_dist_group=sync_dist_group,
sync_dist_op=sync_dist_op,
tbptt_pad_token=tbptt_pad_token,
tbptt_reduce_fx=tbptt_reduce_fx,
)

def get_callback_metrics(self) -> dict:
result = {}
if self.early_stop_on:
result['early_stop_on'] = self.early_stop_on
if self.checkpoint_on:
result['checkpoint_on'] = self.checkpoint_on
return result

def write(self, name: str, values: Union[Tensor, list], filename: str = 'predictions.pt'):
"""Add feature name and value pair to collection of predictions that will be written to disk on
`validation_end` or `test_end`. If running on multiple GPUs, you will get separate `n_gpu`
prediction files with the rank prepended onto filename.
Example::
result = pl.EvalResult()
result.write('ids', [0, 1, 2])
result.write('preds', ['cat', 'dog', 'dog'])
Args:
name: Feature name that will turn into column header of predictions file
values: Flat tensor or list of row values for given feature column 'name'.
filename: Filepath where your predictions will be saved. Defaults to 'predictions.pt'.
"""
# Type check the incoming arguments
if not isinstance(name, str):
raise ValueError(f"Expected str for 'name' but got {type(name)}")
if not isinstance(filename, str):
raise ValueError(f"Expected str for 'filename' but got {type(name)}")

if isinstance(values, Tensor):
values = values.detach()

preds = getattr(self, 'predictions', None)
if preds is None:
self.predictions = {filename: {name: values}}
elif filename not in preds:
preds[filename] = {name: values}
elif name not in preds[filename]:
preds[filename][name] = values
elif isinstance(values, Tensor):
preds[filename][name] = torch.cat((preds[filename][name], values))
elif isinstance(values, list):
preds[filename][name].extend(values)

def write_dict(self, predictions_dict, filename='predictions.pt'):
"""Calls EvalResult.write() for each key-value pair in predictions_dict.
It is recommended that you use this function call instead of .write if you need to
store more than one column of predictions in your output file.
Example::
predictions_to_write = {'preds': ['cat', 'dog'], 'ids': tensor([0, 1])}
result.write_dict(predictions_to_write)
Args:
predictions_dict ([type]): Dict of predictions to store and then write to filename at eval end.
filename (str, optional): File where your predictions will be stored. Defaults to './predictions.pt'.
"""
for k, v in predictions_dict.items():
self.write(k, v, filename)


def weighted_mean(result, weights):

if isinstance(result, dict):
Expand Down
75 changes: 27 additions & 48 deletions pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import torch

from pytorch_lightning.core import memory
from pytorch_lightning.core.step_result import EvalResult, Result
from pytorch_lightning.core.step_result import Result
from pytorch_lightning.loggers import LoggerCollection, TensorBoardLogger
from pytorch_lightning.trainer.connectors.logger_connector.callback_hook_validator import CallbackHookNameValidator
from pytorch_lightning.trainer.connectors.logger_connector.epoch_result_store import EpochResultStore, LoggerStages
Expand Down Expand Up @@ -259,8 +259,8 @@ def add_progress_bar_metrics(self, metrics):

self.trainer.dev_debugger.track_pbar_metrics_history(metrics)

def track_metrics_deprecated(self, deprecated_eval_results, using_eval_result, test_mode):
self._track_callback_metrics(deprecated_eval_results, using_eval_result)
def track_metrics_deprecated(self, deprecated_eval_results, test_mode):
self._track_callback_metrics(deprecated_eval_results)
self.__process_eval_epoch_end_results_and_log_legacy(deprecated_eval_results, test_mode)

def evaluation_epoch_end(self, testing):
Expand Down Expand Up @@ -314,53 +314,41 @@ def get_evaluate_epoch_results(self, test_mode):
self.eval_loop_results = []
return results

def _track_callback_metrics(self, eval_results, using_eval_result):
def _track_callback_metrics(self, eval_results):
if len(eval_results) > 0 and (eval_results[0] is None or not isinstance(eval_results[0], Result)):
return

if using_eval_result:
if isinstance(eval_results, list):
for eval_result in eval_results:
self.trainer.logger_connector.callback_metrics.update(eval_result.callback_metrics)
if self.trainer.testing:
self.trainer.logger_connector.evaluation_callback_metrics.update(
eval_result.callback_metrics)
else:
self.trainer.logger_connector.callback_metrics.update(eval_results.callback_metrics)
if self.trainer.testing:
self.trainer.logger_connector.evaluation_callback_metrics.update(eval_results.callback_metrics)
else:
flat = {}
if isinstance(eval_results, list):
for eval_result in eval_results:
# with a scalar return, auto set it to "val_loss" for callbacks
if isinstance(eval_result, torch.Tensor):
flat = {'val_loss': eval_result}
elif isinstance(eval_result, dict):
flat = flatten_dict(eval_result)

# removing val_loss magic word to map to checkpoint + ES callback
if 'val_loss' in flat:
flat['checkpoint_on'] = flat['val_loss']
flat['early_stop_on'] = flat['val_loss']
self.trainer.logger_connector.callback_metrics.update(flat)
if self.trainer.testing:
self.trainer.logger_connector.evaluation_callback_metrics.update(flat)
else:
flat = {}
if isinstance(eval_results, list):
for eval_result in eval_results:
# with a scalar return, auto set it to "val_loss" for callbacks
if isinstance(eval_results, torch.Tensor):
flat = {'val_loss': eval_results}
else:
flat = flatten_dict(eval_results)
if isinstance(eval_result, torch.Tensor):
flat = {'val_loss': eval_result}
elif isinstance(eval_result, dict):
flat = flatten_dict(eval_result)

# removing val_loss magic word to map to checkpoint + ES callback
if 'val_loss' in flat:
flat['checkpoint_on'] = flat['val_loss']
flat['early_stop_on'] = flat['val_loss']

self.trainer.logger_connector.callback_metrics.update(flat)
if self.trainer.testing:
self.trainer.logger_connector.evaluation_callback_metrics.update(flat)
else:
# with a scalar return, auto set it to "val_loss" for callbacks
if isinstance(eval_results, torch.Tensor):
flat = {'val_loss': eval_results}
else:
flat = flatten_dict(eval_results)

# removing val_loss magic word to map to checkpoint + ES callback
if 'val_loss' in flat:
flat['checkpoint_on'] = flat['val_loss']
flat['early_stop_on'] = flat['val_loss']

self.trainer.logger_connector.callback_metrics.update(flat)
if self.trainer.testing:
self.trainer.logger_connector.evaluation_callback_metrics.update(flat)

def __process_eval_epoch_end_results_and_log_legacy_update(self, prog_bar_metrics, log_metrics, callback_metrics):
# eval loop returns all metrics
Expand Down Expand Up @@ -397,16 +385,7 @@ def __process_eval_epoch_end_results_and_log_legacy(self, eval_results, test_mod
prog_bar_metrics, log_metrics, callback_metrics = {}, {}, {}

for result_idx, result in enumerate(eval_results):
if isinstance(result, EvalResult):
prog_bar_metrics = result.epoch_pbar_metrics
log_metrics = result.epoch_log_metrics
callback_metrics = result.callback_metrics

# in testing we don't need the callback metrics
if test_mode:
callback_metrics = {}
else:
_, prog_bar_metrics, log_metrics, callback_metrics, _ = self.trainer.process_dict_result(result)
_, prog_bar_metrics, log_metrics, callback_metrics, _ = self.trainer.process_dict_result(result)

if num_loaders > 1:
self.__process_eval_epoch_end_results_and_log_legacy_update(
Expand Down
Loading

0 comments on commit 062f090

Please sign in to comment.