From 666b5275c1c06013f76fcf2d6d1ed68145ac1a73 Mon Sep 17 00:00:00 2001 From: Leo Gao <54557097+leogao2@users.noreply.github.com> Date: Mon, 5 Apr 2021 23:11:24 -0600 Subject: [PATCH 1/3] GPTNeo: handle padded wte (#11078) --- src/transformers/models/gpt_neo/modeling_gpt_neo.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/transformers/models/gpt_neo/modeling_gpt_neo.py b/src/transformers/models/gpt_neo/modeling_gpt_neo.py index 9fb0d7475fb9..0aa7639dae86 100755 --- a/src/transformers/models/gpt_neo/modeling_gpt_neo.py +++ b/src/transformers/models/gpt_neo/modeling_gpt_neo.py @@ -112,6 +112,10 @@ def load_tf_weights_in_gpt_neo(model, config, gpt_neo_checkpoint_path): if name[-1] == "w" and name[-2] in ["out_proj", "k_proj", "q_proj", "v_proj", "c_proj", "c_fc"]: array = array.transpose() + if name == ["wte"]: + # if vocab is padded, then trim off the padding embeddings + array = array[:50257] + try: assert ( pointer.shape == array.shape From a58ae40eb2cc4c4aa52b5d6469a79209c6271717 Mon Sep 17 00:00:00 2001 From: Leo Gao <54557097+leogao2@users.noreply.github.com> Date: Mon, 5 Apr 2021 23:31:42 -0600 Subject: [PATCH 2/3] Switch to config.vocab_size --- src/transformers/models/gpt_neo/modeling_gpt_neo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/gpt_neo/modeling_gpt_neo.py b/src/transformers/models/gpt_neo/modeling_gpt_neo.py index 0aa7639dae86..c8da6dd1c2f9 100755 --- a/src/transformers/models/gpt_neo/modeling_gpt_neo.py +++ b/src/transformers/models/gpt_neo/modeling_gpt_neo.py @@ -114,7 +114,7 @@ def load_tf_weights_in_gpt_neo(model, config, gpt_neo_checkpoint_path): if name == ["wte"]: # if vocab is padded, then trim off the padding embeddings - array = array[:50257] + array = array[:config.vocab_size] try: assert ( From cf98c14cedf0dec647ebb155abbbeaa9f0e35309 Mon Sep 17 00:00:00 2001 From: Suraj Patil Date: Wed, 7 Apr 2021 17:22:32 +0530 Subject: [PATCH 3/3] apply review suggestion --- src/transformers/models/gpt_neo/modeling_gpt_neo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/gpt_neo/modeling_gpt_neo.py b/src/transformers/models/gpt_neo/modeling_gpt_neo.py index c8da6dd1c2f9..d6e74d6f89ee 100755 --- a/src/transformers/models/gpt_neo/modeling_gpt_neo.py +++ b/src/transformers/models/gpt_neo/modeling_gpt_neo.py @@ -114,7 +114,7 @@ def load_tf_weights_in_gpt_neo(model, config, gpt_neo_checkpoint_path): if name == ["wte"]: # if vocab is padded, then trim off the padding embeddings - array = array[:config.vocab_size] + array = array[: config.vocab_size] try: assert (