From cd7cf777031ce9aae13d2099a53536af6019064f Mon Sep 17 00:00:00 2001 From: aakbik Date: Fri, 13 Sep 2019 14:13:49 +0200 Subject: [PATCH 1/3] add global variable for FlairEmbeddings optimization --- flair/__init__.py | 3 +++ flair/embeddings.py | 6 +++++- flair/training_utils.py | 5 +++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/flair/__init__.py b/flair/__init__.py index 200f92b090..8a21fc7bf6 100644 --- a/flair/__init__.py +++ b/flair/__init__.py @@ -11,6 +11,9 @@ else: device = torch.device("cpu") +# global variable: optimization_settings +optimization_mode = "default" + from . import data from . import models from . import visual diff --git a/flair/embeddings.py b/flair/embeddings.py index a32083aa9a..c21ca6c07c 100644 --- a/flair/embeddings.py +++ b/flair/embeddings.py @@ -1878,7 +1878,11 @@ def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: if not self.fine_tune: embedding = embedding.detach() - token.set_embedding(self.name, embedding.clone()) + # embedding = embedding.clone() + if flair.optimization_mode == "gpu": + embedding = embedding.clone() + + token.set_embedding(self.name, embedding) all_hidden_states_in_lm = all_hidden_states_in_lm.detach() del all_hidden_states_in_lm diff --git a/flair/training_utils.py b/flair/training_utils.py index 1e8c3fe446..2370bc8ec5 100644 --- a/flair/training_utils.py +++ b/flair/training_utils.py @@ -365,3 +365,8 @@ def store_embeddings(sentences: List[Sentence], storage_mode: str): pin_memory = False if str(flair.device) == "cpu" else True for sentence in sentences: sentence.to("cpu", pin_memory=pin_memory) + + if storage_mode == "gpu": + flair.optimization_mode = "gpu" + else: + flair.optimization_mode = "default" From bbafa747934366327ed0570600f4d5fa9093082d Mon Sep 17 00:00:00 2001 From: aakbik Date: Fri, 13 Sep 2019 14:50:38 +0200 Subject: [PATCH 2/3] GH-1089: global variable for clone() optimization --- flair/embeddings.py | 2 +- flair/training_utils.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/flair/embeddings.py b/flair/embeddings.py index be24c284d1..94a5b39a72 100644 --- a/flair/embeddings.py +++ b/flair/embeddings.py @@ -1878,7 +1878,7 @@ def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: if not self.fine_tune: embedding = embedding.detach() - # embedding = embedding.clone() + # only clone if optimization mode is 'gpu' if flair.optimization_mode == "gpu": embedding = embedding.clone() diff --git a/flair/training_utils.py b/flair/training_utils.py index 2370bc8ec5..941d1d63f9 100644 --- a/flair/training_utils.py +++ b/flair/training_utils.py @@ -366,6 +366,7 @@ def store_embeddings(sentences: List[Sentence], storage_mode: str): for sentence in sentences: sentence.to("cpu", pin_memory=pin_memory) + # if storage mode is GPU, set global variable to let FlairEmbeddings know that optimization is possible if storage_mode == "gpu": flair.optimization_mode = "gpu" else: From 1edf79a30ef0937bd99491cd973be303810e1860 Mon Sep 17 00:00:00 2001 From: aakbik Date: Fri, 13 Sep 2019 16:21:50 +0200 Subject: [PATCH 3/3] GH-1089: consistently rename embedding_storage_mode variable --- flair/__init__.py | 4 ++-- flair/embeddings.py | 2 +- flair/models/sequence_tagger_model.py | 4 ++-- flair/models/similarity_learning_model.py | 6 +++--- flair/models/text_classification_model.py | 4 ++-- flair/models/text_regression_model.py | 4 ++-- flair/nn.py | 4 ++-- flair/trainers/trainer.py | 10 +++++----- flair/training_utils.py | 7 ++----- 9 files changed, 21 insertions(+), 24 deletions(-) diff --git a/flair/__init__.py b/flair/__init__.py index 8a21fc7bf6..33e3fd1dd1 100644 --- a/flair/__init__.py +++ b/flair/__init__.py @@ -11,8 +11,8 @@ else: device = torch.device("cpu") -# global variable: optimization_settings -optimization_mode = "default" +# global variable: embedding_storage_mode +embedding_storage_mode = "default" from . import data from . import models diff --git a/flair/embeddings.py b/flair/embeddings.py index 94a5b39a72..abe14b884f 100644 --- a/flair/embeddings.py +++ b/flair/embeddings.py @@ -1879,7 +1879,7 @@ def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: embedding = embedding.detach() # only clone if optimization mode is 'gpu' - if flair.optimization_mode == "gpu": + if flair.embedding_storage_mode == "gpu": embedding = embedding.clone() token.set_embedding(self.name, embedding) diff --git a/flair/models/sequence_tagger_model.py b/flair/models/sequence_tagger_model.py index 58780e3e5a..7bdb35a389 100644 --- a/flair/models/sequence_tagger_model.py +++ b/flair/models/sequence_tagger_model.py @@ -251,7 +251,7 @@ def evaluate( self, data_loader: DataLoader, out_path: Path = None, - embeddings_storage_mode: str = "none", + embedding_storage_mode: str = "none", ) -> (Result, float): with torch.no_grad(): @@ -320,7 +320,7 @@ def evaluate( else: metric.add_tn(tag) - store_embeddings(batch, embeddings_storage_mode) + store_embeddings(batch, embedding_storage_mode) eval_loss /= batch_no diff --git a/flair/models/similarity_learning_model.py b/flair/models/similarity_learning_model.py index 3535de450f..9b75b06eee 100644 --- a/flair/models/similarity_learning_model.py +++ b/flair/models/similarity_learning_model.py @@ -263,7 +263,7 @@ def evaluate( self, data_loader: DataLoader, out_path: Path = None, - embeddings_storage_mode="none", + embedding_storage_mode="none", ) -> (Result, float): # assumes that for each data pair there's at least one embedding per modality @@ -281,7 +281,7 @@ def evaluate( all_target_embeddings.append( self._embed_target(target_inputs).to(self.eval_device) ) - store_embeddings(data_points, embeddings_storage_mode) + store_embeddings(data_points, embedding_storage_mode) all_target_embeddings = torch.cat(all_target_embeddings, dim=0) # [n0, d0] assert len(target_index) == all_target_embeddings.shape[0] @@ -315,7 +315,7 @@ def evaluate( ] ranks.extend(batch_gt_ranks.tolist()) - store_embeddings(data_points, embeddings_storage_mode) + store_embeddings(data_points, embedding_storage_mode) ranks = np.array(ranks) median_rank = np.median(ranks) diff --git a/flair/models/text_classification_model.py b/flair/models/text_classification_model.py index 97aff58c9f..0030ec01d4 100644 --- a/flair/models/text_classification_model.py +++ b/flair/models/text_classification_model.py @@ -171,7 +171,7 @@ def evaluate( self, data_loader: DataLoader, out_path: Path = None, - embeddings_storage_mode: str = "none", + embedding_storage_mode: str = "none", ) -> (Result, float): with torch.no_grad(): @@ -238,7 +238,7 @@ def evaluate( ): metric.add_tn(label) - store_embeddings(batch, embeddings_storage_mode) + store_embeddings(batch, embedding_storage_mode) eval_loss /= batch_count diff --git a/flair/models/text_regression_model.py b/flair/models/text_regression_model.py index 5dff86eb15..ab94ebcb13 100644 --- a/flair/models/text_regression_model.py +++ b/flair/models/text_regression_model.py @@ -94,7 +94,7 @@ def evaluate( self, data_loader: DataLoader, out_path: Path = None, - embeddings_storage_mode: str = "none", + embedding_storage_mode: str = "none", ) -> (Result, float): with torch.no_grad(): @@ -137,7 +137,7 @@ def evaluate( ) lines.append(eval_line) - store_embeddings(batch, embeddings_storage_mode) + store_embeddings(batch, embedding_storage_mode) eval_loss /= total_count diff --git a/flair/nn.py b/flair/nn.py index 8996068b0f..fd3b24a259 100644 --- a/flair/nn.py +++ b/flair/nn.py @@ -29,13 +29,13 @@ def evaluate( self, data_loader: DataLoader, out_path: Path = None, - embeddings_storage_mode: str = "none", + embedding_storage_mode: str = "none", ) -> (Result, float): """Evaluates the model. Returns a Result object containing evaluation results and a loss value. Implement this to enable evaluation. :param data_loader: DataLoader that iterates over dataset to be evaluated :param out_path: Optional output path to store predictions - :param embeddings_storage_mode: One of 'none', 'cpu' or 'gpu'. 'none' means all embeddings are deleted and + :param embedding_storage_mode: One of 'none', 'cpu' or 'gpu'. 'none' means all embeddings are deleted and freshly recomputed, 'cpu' means all embeddings are stored on CPU, or 'gpu' means all embeddings are stored on GPU :return: Returns a Tuple consisting of a Result object and a loss float value """ diff --git a/flair/trainers/trainer.py b/flair/trainers/trainer.py index 89b081470a..9791739832 100644 --- a/flair/trainers/trainer.py +++ b/flair/trainers/trainer.py @@ -327,7 +327,7 @@ def train( batch_size=eval_mini_batch_size, num_workers=num_workers, ), - embeddings_storage_mode=embeddings_storage_mode, + embedding_storage_mode=embeddings_storage_mode, ) result_line += f"\t{train_eval_result.log_line}" @@ -341,7 +341,7 @@ def train( batch_size=eval_mini_batch_size, num_workers=num_workers, ), - embeddings_storage_mode=embeddings_storage_mode, + embedding_storage_mode=embeddings_storage_mode, ) result_line += f"\t{dev_loss}\t{dev_eval_result.log_line}" log.info( @@ -371,7 +371,7 @@ def train( num_workers=num_workers, ), base_path / "test.tsv", - embeddings_storage_mode=embeddings_storage_mode, + embedding_storage_mode=embeddings_storage_mode, ) result_line += f"\t{test_loss}\t{test_eval_result.log_line}" log.info( @@ -511,7 +511,7 @@ def final_test( num_workers=num_workers, ), out_path=base_path / "test.tsv", - embeddings_storage_mode="none", + embedding_storage_mode="none", ) test_results: Result = test_results @@ -530,7 +530,7 @@ def final_test( num_workers=num_workers, ), out_path=base_path / f"{subcorpus.name}-test.tsv", - embeddings_storage_mode="none", + embedding_storage_mode="none", ) # get and return the final test score of best model diff --git a/flair/training_utils.py b/flair/training_utils.py index 941d1d63f9..abb2afb807 100644 --- a/flair/training_utils.py +++ b/flair/training_utils.py @@ -366,8 +366,5 @@ def store_embeddings(sentences: List[Sentence], storage_mode: str): for sentence in sentences: sentence.to("cpu", pin_memory=pin_memory) - # if storage mode is GPU, set global variable to let FlairEmbeddings know that optimization is possible - if storage_mode == "gpu": - flair.optimization_mode = "gpu" - else: - flair.optimization_mode = "default" + # record current embedding storage mode to allow optimization (for instance in FlairEmbeddings class) + flair.embedding_storage_mode = storage_mode