From 3b3d69de08b53e9918af832a5ca6bbe1713ddf2c Mon Sep 17 00:00:00 2001 From: Julien Plu Date: Wed, 27 Jan 2021 09:36:49 +0100 Subject: [PATCH] Add a test for mixed precision (#9806) --- tests/test_modeling_tf_albert.py | 4 ++++ tests/test_modeling_tf_bart.py | 4 ++++ tests/test_modeling_tf_blenderbot.py | 4 ++++ tests/test_modeling_tf_blenderbot_small.py | 4 ++++ tests/test_modeling_tf_common.py | 14 ++++++++++++++ tests/test_modeling_tf_ctrl.py | 4 ++++ tests/test_modeling_tf_flaubert.py | 8 ++------ tests/test_modeling_tf_funnel.py | 8 ++++++++ tests/test_modeling_tf_gpt2.py | 4 ++++ tests/test_modeling_tf_led.py | 4 ++++ tests/test_modeling_tf_longformer.py | 15 +++++++++++++-- tests/test_modeling_tf_lxmert.py | 4 ++++ tests/test_modeling_tf_marian.py | 4 ++++ tests/test_modeling_tf_mbart.py | 4 ++++ tests/test_modeling_tf_mobilebert.py | 4 ++++ tests/test_modeling_tf_openai.py | 4 ++++ tests/test_modeling_tf_pegasus.py | 4 ++++ tests/test_modeling_tf_t5.py | 8 ++++++++ tests/test_modeling_tf_transfo_xl.py | 4 ++++ tests/test_modeling_tf_xlm.py | 4 ++++ 20 files changed, 105 insertions(+), 8 deletions(-) diff --git a/tests/test_modeling_tf_albert.py b/tests/test_modeling_tf_albert.py index d037738081b4..e020cc5cf787 100644 --- a/tests/test_modeling_tf_albert.py +++ b/tests/test_modeling_tf_albert.py @@ -294,6 +294,10 @@ def test_model_common_attributes(self): name = model.get_bias() assert name is None + def test_mixed_precision(self): + # TODO JP: Make ALBERT float16 compliant + pass + @slow def test_model_from_pretrained(self): for model_name in TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: diff --git a/tests/test_modeling_tf_bart.py b/tests/test_modeling_tf_bart.py index 6a0110d4a6d6..367b65dfb693 100644 --- a/tests/test_modeling_tf_bart.py +++ b/tests/test_modeling_tf_bart.py @@ -278,6 +278,10 @@ def test_saved_model_creation(self): # This test is too long (>30sec) and makes fail the CI pass + def test_mixed_precision(self): + # TODO JP: Make BART float16 compliant + pass + def _assert_tensors_equal(a, b, atol=1e-12, prefix=""): """If tensors not close, or a and b arent both tensors, raise a nice Assertion error.""" diff --git a/tests/test_modeling_tf_blenderbot.py b/tests/test_modeling_tf_blenderbot.py index c38cd66efb2b..757f44fb9653 100644 --- a/tests/test_modeling_tf_blenderbot.py +++ b/tests/test_modeling_tf_blenderbot.py @@ -214,6 +214,10 @@ def test_saved_model_creation(self): # This test is too long (>30sec) and makes fail the CI pass + def test_mixed_precision(self): + # TODO JP: Make Blenderbot float16 compliant + pass + def test_resize_token_embeddings(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/tests/test_modeling_tf_blenderbot_small.py b/tests/test_modeling_tf_blenderbot_small.py index 8b322a7b7c87..c2454964a0a2 100644 --- a/tests/test_modeling_tf_blenderbot_small.py +++ b/tests/test_modeling_tf_blenderbot_small.py @@ -279,6 +279,10 @@ def test_saved_model_creation(self): # This test is too long (>30sec) and makes fail the CI pass + def test_mixed_precision(self): + # TODO JP: Make Blenderbot Small float16 compliant + pass + def _assert_tensors_equal(a, b, atol=1e-12, prefix=""): """If tensors not close, or a and b arent both tensors, raise a nice Assertion error.""" diff --git a/tests/test_modeling_tf_common.py b/tests/test_modeling_tf_common.py index f41c4b146972..74250cee612a 100644 --- a/tests/test_modeling_tf_common.py +++ b/tests/test_modeling_tf_common.py @@ -279,6 +279,20 @@ def test_saved_model_with_attentions_output(self): [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length], ) + def test_mixed_precision(self): + tf.keras.mixed_precision.experimental.set_policy("mixed_float16") + + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + class_inputs_dict = self._prepare_for_class(inputs_dict, model_class) + model = model_class(config) + outputs = model(class_inputs_dict) + + self.assertIsNotNone(outputs) + + tf.keras.mixed_precision.experimental.set_policy("float32") + def test_keras_save_load(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/tests/test_modeling_tf_ctrl.py b/tests/test_modeling_tf_ctrl.py index 5664c968ce03..f870edfdc5b8 100644 --- a/tests/test_modeling_tf_ctrl.py +++ b/tests/test_modeling_tf_ctrl.py @@ -221,6 +221,10 @@ def test_model_common_attributes(self): name = model.get_bias() assert name is None + def test_mixed_precision(self): + # TODO JP: Make CTRL float16 compliant + pass + @slow def test_model_from_pretrained(self): for model_name in TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: diff --git a/tests/test_modeling_tf_flaubert.py b/tests/test_modeling_tf_flaubert.py index 640dd9cfe863..27be521b9881 100644 --- a/tests/test_modeling_tf_flaubert.py +++ b/tests/test_modeling_tf_flaubert.py @@ -330,12 +330,8 @@ def test_model_from_pretrained(self): model = TFFlaubertModel.from_pretrained(model_name) self.assertIsNotNone(model) - def test_saved_model_with_hidden_states_output(self): - # Should be uncommented during patrick TF refactor - pass - - def test_saved_model_with_attentions_output(self): - # Should be uncommented during patrick TF refactor + def test_mixed_precision(self): + # TODO JP: Make Flaubert float16 compliant pass diff --git a/tests/test_modeling_tf_funnel.py b/tests/test_modeling_tf_funnel.py index a530bd59a0e2..e0d50aeb14f9 100644 --- a/tests/test_modeling_tf_funnel.py +++ b/tests/test_modeling_tf_funnel.py @@ -371,6 +371,10 @@ def test_saved_model_creation(self): # This test is too long (>30sec) and makes fail the CI pass + def test_mixed_precision(self): + # TODO JP: Make Funnel float16 compliant + pass + @require_tf class TFFunnelBaseModelTest(TFModelTesterMixin, unittest.TestCase): @@ -401,3 +405,7 @@ def test_for_multiple_choice(self): def test_saved_model_creation(self): # This test is too long (>30sec) and makes fail the CI pass + + def test_mixed_precision(self): + # TODO JP: Make Funnel float16 compliant + pass diff --git a/tests/test_modeling_tf_gpt2.py b/tests/test_modeling_tf_gpt2.py index 321f87b5e9e3..56ed81643d4d 100644 --- a/tests/test_modeling_tf_gpt2.py +++ b/tests/test_modeling_tf_gpt2.py @@ -387,6 +387,10 @@ def test_gpt2_sequence_classification_model(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_gpt2_for_sequence_classification(*config_and_inputs) + def test_mixed_precision(self): + # TODO JP: Make GPT2 float16 compliant + pass + @slow def test_model_from_pretrained(self): for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: diff --git a/tests/test_modeling_tf_led.py b/tests/test_modeling_tf_led.py index c4f611be6a51..6447bd0f13a3 100644 --- a/tests/test_modeling_tf_led.py +++ b/tests/test_modeling_tf_led.py @@ -357,6 +357,10 @@ def test_saved_model_creation(self): # This test is too long (>30sec) and makes fail the CI pass + def test_mixed_precision(self): + # TODO JP: Make LED float16 compliant + pass + def test_saved_model_with_attentions_output(self): # This test don't pass because of the error: # condition [13,8,4,5], then [13,8,4,5], and else [13,8,4,6] must be broadcastable diff --git a/tests/test_modeling_tf_longformer.py b/tests/test_modeling_tf_longformer.py index b56002ed050c..43b32e952486 100644 --- a/tests/test_modeling_tf_longformer.py +++ b/tests/test_modeling_tf_longformer.py @@ -340,14 +340,25 @@ def test_for_multiple_choice(self): @slow def test_saved_model_with_attentions_output(self): - # longformer has special attentions which are not - # compatible in graph mode + # This test don't pass because of the error: + # condition [13,8,4,5], then [13,8,4,5], and else [13,8,4,6] must be broadcastable + # This occurs line 323 in modeling_tf_led.py because the condition line 255 + # returns a tensor of shape + # [batch_size, seq_len, self.num_heads, self.one_sided_attn_window_size * 2 + 2] + # if is_global_attn is True and a tensor of shape + # [batch_size, seq_len, self.num_heads, self.one_sided_attn_window_size * 2 + 1] + # This is due to the tf.concat call line 703 that adds one dimension + # Need to check with PVP how to properly fix this pass def test_saved_model_creation(self): # This test is too long (>30sec) and makes fail the CI pass + def test_mixed_precision(self): + # TODO JP: Make Longformer float16 compliant + pass + @require_tf @require_sentencepiece diff --git a/tests/test_modeling_tf_lxmert.py b/tests/test_modeling_tf_lxmert.py index 496fb1df2671..31afc66ecb26 100644 --- a/tests/test_modeling_tf_lxmert.py +++ b/tests/test_modeling_tf_lxmert.py @@ -704,6 +704,10 @@ def test_saved_model_creation(self): # This test is too long (>30sec) and makes fail the CI pass + def test_mixed_precision(self): + # TODO JP: Make Lxmert float16 compliant + pass + @slow def test_saved_model_with_hidden_states_output(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py index 917c5f5fb3dd..1a58386a70a1 100644 --- a/tests/test_modeling_tf_marian.py +++ b/tests/test_modeling_tf_marian.py @@ -247,6 +247,10 @@ def test_saved_model_creation(self): # This test is too long (>30sec) and makes fail the CI pass + def test_mixed_precision(self): + # TODO JP: Make Marian float16 compliant + pass + def test_resize_token_embeddings(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/tests/test_modeling_tf_mbart.py b/tests/test_modeling_tf_mbart.py index a3b6cd416b2b..766773973e2f 100644 --- a/tests/test_modeling_tf_mbart.py +++ b/tests/test_modeling_tf_mbart.py @@ -218,6 +218,10 @@ def test_saved_model_creation(self): # This test is too long (>30sec) and makes fail the CI pass + def test_mixed_precision(self): + # TODO JP: Make MBart float16 compliant + pass + def test_resize_token_embeddings(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/tests/test_modeling_tf_mobilebert.py b/tests/test_modeling_tf_mobilebert.py index cf918f9b30d4..35c617de3b73 100644 --- a/tests/test_modeling_tf_mobilebert.py +++ b/tests/test_modeling_tf_mobilebert.py @@ -309,6 +309,10 @@ def test_saved_model_creation(self): # This test is too long (>30sec) and makes fail the CI pass + def test_mixed_precision(self): + # TODO JP: Make MobileBert float16 compliant + pass + @slow def test_model_from_pretrained(self): # for model_name in TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: diff --git a/tests/test_modeling_tf_openai.py b/tests/test_modeling_tf_openai.py index 49689a0e4d46..8d8c21835a9c 100644 --- a/tests/test_modeling_tf_openai.py +++ b/tests/test_modeling_tf_openai.py @@ -245,6 +245,10 @@ def test_openai_gpt_sequence_classification_model(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*config_and_inputs) + def test_mixed_precision(self): + # TODO JP: Make OpenAIGPT float16 compliant + pass + @slow def test_model_from_pretrained(self): for model_name in TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: diff --git a/tests/test_modeling_tf_pegasus.py b/tests/test_modeling_tf_pegasus.py index 63dc33604405..a7a5b03f950b 100644 --- a/tests/test_modeling_tf_pegasus.py +++ b/tests/test_modeling_tf_pegasus.py @@ -245,6 +245,10 @@ def test_saved_model_creation(self): # This test is too long (>30sec) and makes fail the CI pass + def test_mixed_precision(self): + # TODO JP: Make Pegasus float16 compliant + pass + def test_resize_token_embeddings(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/tests/test_modeling_tf_t5.py b/tests/test_modeling_tf_t5.py index f4a235f8feec..a01273fc22b5 100644 --- a/tests/test_modeling_tf_t5.py +++ b/tests/test_modeling_tf_t5.py @@ -306,6 +306,10 @@ def test_saved_model_creation(self): # This test is too long (>30sec) and makes fail the CI pass + def test_mixed_precision(self): + # TODO JP: Make T5 float16 compliant + pass + @slow def test_model_from_pretrained(self): model = TFT5Model.from_pretrained("t5-small") @@ -435,6 +439,10 @@ def test_model(self): def test_train_pipeline_custom_model(self): pass + def test_mixed_precision(self): + # TODO JP: Make T5 float16 compliant + pass + @require_tf @require_sentencepiece diff --git a/tests/test_modeling_tf_transfo_xl.py b/tests/test_modeling_tf_transfo_xl.py index 64cc315c37f9..86d9468cec52 100644 --- a/tests/test_modeling_tf_transfo_xl.py +++ b/tests/test_modeling_tf_transfo_xl.py @@ -204,6 +204,10 @@ def test_model_common_attributes(self): name = model.get_bias() assert name is None + def test_mixed_precision(self): + # TODO JP: Make TransfoXL float16 compliant + pass + @slow def test_model_from_pretrained(self): for model_name in TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: diff --git a/tests/test_modeling_tf_xlm.py b/tests/test_modeling_tf_xlm.py index c6a2d6c94a5c..466f5640db90 100644 --- a/tests/test_modeling_tf_xlm.py +++ b/tests/test_modeling_tf_xlm.py @@ -326,6 +326,10 @@ def test_for_multiple_choice(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xlm_for_multiple_choice(*config_and_inputs) + def test_mixed_precision(self): + # TODO JP: Make XLM float16 compliant + pass + @slow def test_model_from_pretrained(self): for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: