diff --git a/crates/polars-core/src/series/any_value.rs b/crates/polars-core/src/series/any_value.rs
index 530983313bb0..3a9ea572a6eb 100644
--- a/crates/polars-core/src/series/any_value.rs
+++ b/crates/polars-core/src/series/any_value.rs
@@ -494,11 +494,13 @@ fn any_values_to_list(
     #[allow(unused_mut)]
     let mut out: ListChunked = if inner_type == &DataType::Null {
         let mut first_value_dtype = DataType::Null;
+        let mut set_first = false;
         avs.iter()
             .map(|av| match av {
                 AnyValue::List(b) => {
-                    if !b.is_empty() && matches!(first_value_dtype, DataType::Null) {
+                    if !set_first && first_value_dtype != *b.dtype() {
                         first_value_dtype = b.dtype().clone();
+                        set_first = true;
                     } else if b.is_empty() || b.null_count() == b.len() {
                         return b.cast(&first_value_dtype).ok();
                     }