diff --git a/src/transformers/models/convbert/modeling_tf_convbert.py b/src/transformers/models/convbert/modeling_tf_convbert.py
index fbd9ecb4f49e..59e0c3362b75 100644
--- a/src/transformers/models/convbert/modeling_tf_convbert.py
+++ b/src/transformers/models/convbert/modeling_tf_convbert.py
@@ -435,9 +435,10 @@ def build(self, input_shape):
         )
 
     def call(self, hidden_states):
-        batch_size = shape_list(tensor=hidden_states)[1]
-        x = tf.reshape(hidden_states, [-1, self.num_groups, self.group_in_dim])
-        x = tf.matmul(a=x, b=self.kernel, transpose_b=True)
+        batch_size = shape_list(hidden_states)[0]
+        x = tf.transpose(tf.reshape(hidden_states, [-1, self.num_groups, self.group_in_dim]), [1, 0, 2])
+        x = tf.matmul(x, self.kernel)
+        x = tf.transpose(x, [1, 0, 2])
         x = tf.reshape(x, [batch_size, -1, self.output_size])
         x = tf.nn.bias_add(value=x, bias=self.bias)
         return x