diff --git a/keras_nlp/layers/modeling/token_and_position_embedding.py b/keras_nlp/layers/modeling/token_and_position_embedding.py
index 019efe9980..ad89af05f9 100644
--- a/keras_nlp/layers/modeling/token_and_position_embedding.py
+++ b/keras_nlp/layers/modeling/token_and_position_embedding.py
@@ -62,7 +62,7 @@ def __init__(
         sequence_length,
         embedding_dim,
         tie_weights=True,
-        embeddings_initializer="glorot_uniform",
+        embeddings_initializer="uniform",
         mask_zero=False,
         **kwargs
     ):