generalize toListImpl to support conversions into lists and sequences

The previous implementation already was based on iterating over rows, with this generalization, the user can now decide whether to eagerly convert a dataframe into a list, or use lazy transformations via a sequence.
Kotlin · Jan 31, 2025 · 9781085 · 9781085
1 parent bddf7bf
commit 9781085
Show file tree

Hide file tree

Showing 8 changed files with 54 additions and 15 deletions.
diff --git a/core/api/core.api b/core/api/core.api
@@ -9808,8 +9808,8 @@ public final class org/jetbrains/kotlinx/dataframe/impl/api/ToDataFrameKt {
 	public static final fun createDataFrameImpl (Ljava/lang/Iterable;Lkotlin/reflect/KClass;Lkotlin/jvm/functions/Function1;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
 }
 
-public final class org/jetbrains/kotlinx/dataframe/impl/api/ToListKt {
-	public static final fun toListImpl (Lorg/jetbrains/kotlinx/dataframe/DataFrame;Lkotlin/reflect/KType;)Ljava/util/List;
+public final class org/jetbrains/kotlinx/dataframe/impl/api/ToSequenceKt {
+	public static final fun toSequenceImpl (Lorg/jetbrains/kotlinx/dataframe/DataFrame;Lkotlin/reflect/KType;)Ljava/lang/Iterable;
 }
 
 public final class org/jetbrains/kotlinx/dataframe/impl/api/UpdateKt {

diff --git a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/toList.kt b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/toList.kt
@@ -2,13 +2,13 @@ package org.jetbrains.kotlinx.dataframe.api
 
 import org.jetbrains.kotlinx.dataframe.AnyFrame
 import org.jetbrains.kotlinx.dataframe.DataFrame
-import org.jetbrains.kotlinx.dataframe.impl.api.toListImpl
+import org.jetbrains.kotlinx.dataframe.impl.api.toSequenceImpl
 import kotlin.reflect.typeOf
 
 // region DataFrame
 
-public inline fun <reified T> DataFrame<T>.toList(): List<T> = toListImpl(typeOf<T>()) as List<T>
+public inline fun <reified T> DataFrame<T>.toList(): List<T> = toSequenceImpl(typeOf<T>()).toList() as List<T>
 
-public inline fun <reified T> AnyFrame.toListOf(): List<T> = toListImpl(typeOf<T>()) as List<T>
+public inline fun <reified T> AnyFrame.toListOf(): List<T> = toSequenceImpl(typeOf<T>()).toList() as List<T>
 
 // endregion
diff --git a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/toSequence.kt b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/toSequence.kt
@@ -0,0 +1,14 @@
+package org.jetbrains.kotlinx.dataframe.api
+
+import org.jetbrains.kotlinx.dataframe.AnyFrame
+import org.jetbrains.kotlinx.dataframe.DataFrame
+import org.jetbrains.kotlinx.dataframe.impl.api.toSequenceImpl
+import kotlin.reflect.typeOf
+
+// region DataFrame
+
+public inline fun <reified T> DataFrame<T>.toSequence(): Sequence<T> = toSequenceImpl(typeOf<T>()) as Sequence<T>
+
+public inline fun <reified T> AnyFrame.toSequenceOf(): Sequence<T> = toSequenceImpl(typeOf<T>()) as Sequence<T>
+
+// endregion
diff --git a/...ains/kotlinx/dataframe/impl/api/toList.kt → .../kotlinx/dataframe/impl/api/toSequence.kt b/...ains/kotlinx/dataframe/impl/api/toList.kt → .../kotlinx/dataframe/impl/api/toSequence.kt
@@ -21,7 +21,7 @@ import kotlin.reflect.full.withNullability
 import kotlin.reflect.jvm.jvmErasure
 
 @PublishedApi
-internal fun AnyFrame.toListImpl(type: KType): List<Any> {
+internal fun AnyFrame.toSequenceImpl(type: KType): Iterable<Any> {
     val clazz = type.jvmErasure
     require(clazz.isData) { "`$clazz` is not a data class. `toList` is supported only for data classes." }
 
@@ -46,15 +46,15 @@ internal fun AnyFrame.toListImpl(type: KType): List<Any> {
                     val col: AnyCol = if (it.type.jvmErasure == List::class) {
                         val elementType = it.type.arguments[0].type
                         require(elementType != null) { "FrameColumn can not be converted to type `List<*>`" }
-                        column.asAnyFrameColumn().map { it.toListImpl(elementType) }
+                        column.asAnyFrameColumn().map { it.toSequenceImpl(elementType).toList() }
                     } else {
                         error("FrameColumn can not be converted to type `${it.type}`")
                     }
                     col
                 }
 
                 ColumnKind.Group -> {
-                    DataColumn.createValueColumn(column.name(), column.asColumnGroup().toListImpl(it.type))
+                    DataColumn.createValueColumn(column.name(), column.asColumnGroup().toSequenceImpl(it.type).toList())
                 }
 
                 ColumnKind.Value -> {

diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/toList.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/toList.kt
@@ -2,13 +2,13 @@ package org.jetbrains.kotlinx.dataframe.api
 
 import org.jetbrains.kotlinx.dataframe.AnyFrame
 import org.jetbrains.kotlinx.dataframe.DataFrame
-import org.jetbrains.kotlinx.dataframe.impl.api.toListImpl
+import org.jetbrains.kotlinx.dataframe.impl.api.toSequenceImpl
 import kotlin.reflect.typeOf
 
 // region DataFrame
 
-public inline fun <reified T> DataFrame<T>.toList(): List<T> = toListImpl(typeOf<T>()) as List<T>
+public inline fun <reified T> DataFrame<T>.toList(): List<T> = toSequenceImpl(typeOf<T>()).toList() as List<T>
 
-public inline fun <reified T> AnyFrame.toListOf(): List<T> = toListImpl(typeOf<T>()) as List<T>
+public inline fun <reified T> AnyFrame.toListOf(): List<T> = toSequenceImpl(typeOf<T>()).toList() as List<T>
 
 // endregion
diff --git a/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/toSequence.kt b/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/toSequence.kt
@@ -0,0 +1,14 @@
+package org.jetbrains.kotlinx.dataframe.api
+
+import org.jetbrains.kotlinx.dataframe.AnyFrame
+import org.jetbrains.kotlinx.dataframe.DataFrame
+import org.jetbrains.kotlinx.dataframe.impl.api.toSequenceImpl
+import kotlin.reflect.typeOf
+
+// region DataFrame
+
+public inline fun <reified T> DataFrame<T>.toSequence(): Sequence<T> = toSequenceImpl(typeOf<T>()) as Sequence<T>
+
+public inline fun <reified T> AnyFrame.toSequenceOf(): Sequence<T> = toSequenceImpl(typeOf<T>()) as Sequence<T>
+
+// endregion
diff --git a/...ains/kotlinx/dataframe/impl/api/toList.kt → .../kotlinx/dataframe/impl/api/toSequence.kt b/...ains/kotlinx/dataframe/impl/api/toList.kt → .../kotlinx/dataframe/impl/api/toSequence.kt
@@ -21,7 +21,7 @@ import kotlin.reflect.full.withNullability
 import kotlin.reflect.jvm.jvmErasure
 
 @PublishedApi
-internal fun AnyFrame.toListImpl(type: KType): List<Any> {
+internal fun AnyFrame.toSequenceImpl(type: KType): Iterable<Any> {
     val clazz = type.jvmErasure
     require(clazz.isData) { "`$clazz` is not a data class. `toList` is supported only for data classes." }
 
@@ -46,15 +46,15 @@ internal fun AnyFrame.toListImpl(type: KType): List<Any> {
                     val col: AnyCol = if (it.type.jvmErasure == List::class) {
                         val elementType = it.type.arguments[0].type
                         require(elementType != null) { "FrameColumn can not be converted to type `List<*>`" }
-                        column.asAnyFrameColumn().map { it.toListImpl(elementType) }
+                        column.asAnyFrameColumn().map { it.toSequenceImpl(elementType).toList() }
                     } else {
                         error("FrameColumn can not be converted to type `${it.type}`")
                     }
                     col
                 }
 
                 ColumnKind.Group -> {
-                    DataColumn.createValueColumn(column.name(), column.asColumnGroup().toListImpl(it.type))
+                    DataColumn.createValueColumn(column.name(), column.asColumnGroup().toSequenceImpl(it.type).toList())
                 }
 
                 ColumnKind.Value -> {

diff --git a/docs/StardustDocs/topics/collectionsInterop.md b/docs/StardustDocs/topics/collectionsInterop.md
@@ -82,7 +82,7 @@ for Gradle or the [Kotlin Jupyter kernel](gettingStartedJupyterNotebook.md)
 
 </tip>
 
-After your data is transformed, [`DataFrame`](DataFrame.md) instances can be exported 
+After your data is transformed, [`DataFrame`](DataFrame.md) instances can be exported eagerly
 into [`List`](https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.collections/-list/) of another data class using [toList](toList.md) or [toListOf](toList.md#tolistof) extensions:
 
 <!---FUN listInterop4-->
@@ -93,6 +93,17 @@ data class Output(val a: Int, val b: Int, val c: Int)
 val result = df2.toListOf<Output>()
 ```
 
+Alternatively, one can create lazy [`Sequence`](https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.collections/-sequence/) objects.
+This avoids holding the entire list of objects in memory as objects are created on the fly as needed.
+
+<!---FUN listInterop5-->
+
+```kotlin
+data class Output(val a: Int, val b: Int, val c: Int)
+
+val result = df2.toSequenceOf<Output>()
+```
+
 <!---END-->
 
 ### Converting columns with object instances to ColumnGroup