fix embedding layernorm logic and default in config -- corrected

OATML · Apr 25, 2022 · a399360 · a399360
1 parent ab90fed
commit a399360
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 6 deletions.
diff --git a/npt/configs.py b/npt/configs.py
@@ -483,7 +483,7 @@ def build_parser():
     # #### Normalization ######################################################
 
     parser.add_argument(
-        '--model_embedding_layer_norm', default=True, type='bool',
+        '--model_embedding_layer_norm', default=False, type='bool',
         help='(Disable) use of layer normalization after in-/out-embedding.')
     parser.add_argument(
         '--model_att_block_layer_norm', default=True, type='bool',

diff --git a/npt/model/npt.py b/npt/model/npt.py
@@ -142,11 +142,11 @@ def __init__(self, c, metadata, device=None):
             if c.model_hidden_dropout_prob else None)
 
         # LayerNorm applied after embedding, before dropout
-        if self.c.embedding_layer_norm and device is None:
+        if self.c.model_embedding_layer_norm and device is None:
             print(
                 'Must provide a device in NPT initialization with embedding '
                 'LayerNorm.')
-        elif self.c.embedding_layer_norm:
+        elif self.c.model_embedding_layer_norm:
             # we batch over rows and columns
             # (i.e. just normalize over E)
             layer_norm_dims = [self.dim_hidden]
@@ -155,9 +155,6 @@ def __init__(self, c, metadata, device=None):
         else:
             self.embedding_layer_norm = None
 
-        if True:
-            a = 1
-
         # *** Input In/Out Embeddings ***
         # Don't use for Image Patching - those are handled by the respective
         # init_image_patching