Merge branch 'refs/heads/master' into kdoc-fixes

Author: Jolanrensen

.github/workflows/gradle-wrapper-validation.yml

@@ -0,0 +1,10 @@
+name: "Validate Gradle Wrapper"
+on: [push, pull_request]
+
+jobs:
+  validation:
+    name: "Validation"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: gradle/wrapper-validation-action@v2

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/pivot.kt

@@ -148,7 +148,7 @@ public fun <G> GroupBy<*, G>.pivotCounts(vararg columns: KProperty<*>, inward: B
 
 // region pivot
 
-public fun <T> AggregateGroupedDsl<T>.pivot(inward: Boolean = true, columns: ColumnsSelector<T, *>): PivotGroupBy<T> =
+public fun <T> AggregateGroupedDsl<T>.pivot(inward: Boolean = true, columns: PivotColumnsSelector<T, *>): PivotGroupBy<T> =
     PivotInAggregateImpl(this, columns, inward)
 
 public fun <T> AggregateGroupedDsl<T>.pivot(vararg columns: String, inward: Boolean = true): PivotGroupBy<T> =

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/shuffle.kt

@@ -3,15 +3,20 @@ package org.jetbrains.kotlinx.dataframe.api
 import org.jetbrains.kotlinx.dataframe.DataColumn
 import org.jetbrains.kotlinx.dataframe.DataFrame
 import org.jetbrains.kotlinx.dataframe.indices
+import kotlin.random.Random
 
 // region DataColumn
 
+public fun <T> DataColumn<T>.shuffle(random: Random): DataColumn<T> = get(indices.shuffled(random))
+
 public fun <T> DataColumn<T>.shuffle(): DataColumn<T> = get(indices.shuffled())
 
 // endregion
 
 // region DataFrame
 
+public fun <T> DataFrame<T>.shuffle(random: Random): DataFrame<T> = getRows(indices.shuffled(random))
+
 public fun <T> DataFrame<T>.shuffle(): DataFrame<T> = getRows(indices.shuffled())
 
 // endregion

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/aggregation/PivotInAggregateImpl.kt

@@ -4,14 +4,15 @@ import org.jetbrains.kotlinx.dataframe.ColumnsSelector
 import org.jetbrains.kotlinx.dataframe.DataFrame
 import org.jetbrains.kotlinx.dataframe.aggregation.AggregateBody
 import org.jetbrains.kotlinx.dataframe.aggregation.AggregateGroupedDsl
+import org.jetbrains.kotlinx.dataframe.api.PivotColumnsSelector
 import org.jetbrains.kotlinx.dataframe.api.PivotGroupBy
 import org.jetbrains.kotlinx.dataframe.impl.api.AggregatedPivot
 import org.jetbrains.kotlinx.dataframe.impl.api.aggregatePivot
 import org.jetbrains.kotlinx.dataframe.impl.columns.toColumnSet
 
 internal data class PivotInAggregateImpl<T>(
     val aggregator: AggregateGroupedDsl<T>,
-    val columns: ColumnsSelector<T, *>,
+    val columns: PivotColumnsSelector<T, *>,
     val inward: Boolean?,
     val default: Any? = null
 ) : PivotGroupBy<T>, AggregatableInternal<T> {

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/api/parse.kt

@@ -248,10 +248,10 @@ internal object Parsers : GlobalParserOptions {
             parser
         },
 
-        // kotlin.time.duration
+        // kotlin.time.Duration
         stringParser { catchSilent { Duration.parse(it) } },
 
-        // java.time.duration
+        // java.time.Duration
         stringParser { catchSilent { java.time.Duration.parse(it) } },
 
         // java.time.LocalTime

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/Integration.kt

@@ -45,15 +45,8 @@ import org.jetbrains.kotlinx.dataframe.impl.renderType
 import org.jetbrains.kotlinx.dataframe.io.DataFrameHtmlData
 import org.jetbrains.kotlinx.dataframe.io.SupportedCodeGenerationFormat
 import org.jetbrains.kotlinx.dataframe.io.supportedFormats
-import org.jetbrains.kotlinx.jupyter.api.HTML
-import org.jetbrains.kotlinx.jupyter.api.JupyterClientType
-import org.jetbrains.kotlinx.jupyter.api.KotlinKernelHost
-import org.jetbrains.kotlinx.jupyter.api.Notebook
-import org.jetbrains.kotlinx.jupyter.api.VariableName
-import org.jetbrains.kotlinx.jupyter.api.declare
-import org.jetbrains.kotlinx.jupyter.api.libraries.ColorScheme
-import org.jetbrains.kotlinx.jupyter.api.libraries.JupyterIntegration
-import org.jetbrains.kotlinx.jupyter.api.libraries.resources
+import org.jetbrains.kotlinx.jupyter.api.*
+import org.jetbrains.kotlinx.jupyter.api.libraries.*
 import kotlin.reflect.KClass
 import kotlin.reflect.KProperty
 import kotlin.reflect.KType
@@ -281,16 +274,25 @@ internal class Integration(
         import("org.jetbrains.kotlinx.dataframe.dataTypes.*")
         import("org.jetbrains.kotlinx.dataframe.impl.codeGen.urlCodeGenReader")
 
-        updateVariable<Any> { instance, property ->
-            when (instance) {
-                is AnyCol -> updateAnyColVariable(instance, property, codeGen)
-                is ColumnGroup<*> -> updateColumnGroupVariable(instance, property, codeGen)
-                is AnyRow -> updateAnyRowVariable(instance, property, codeGen)
-                is AnyFrame -> updateAnyFrameVariable(instance, property, codeGen)
-                is ImportDataSchema -> updateImportDataSchemaVariable(instance, property)
-                else -> null
+        addTypeConverter(object : FieldHandler {
+            override val execution = FieldHandlerFactory.createUpdateExecution<Any> { instance, property ->
+                when (instance) {
+                    is AnyCol -> updateAnyColVariable(instance, property, codeGen)
+                    is ColumnGroup<*> -> updateColumnGroupVariable(instance, property, codeGen)
+                    is AnyRow -> updateAnyRowVariable(instance, property, codeGen)
+                    is AnyFrame -> updateAnyFrameVariable(instance, property, codeGen)
+                    is ImportDataSchema -> updateImportDataSchemaVariable(instance, property)
+                    else -> error("${instance::class} should not be handled by Dataframe field handler")
+                }
             }
-        }
+            override fun accepts(value: Any?, property: KProperty<*>): Boolean {
+                return value is AnyCol ||
+                    value is ColumnGroup<*> ||
+                    value is AnyRow ||
+                    value is AnyFrame ||
+                    value is ImportDataSchema
+            }
+        })
 
         fun KotlinKernelHost.addDataSchemas(classes: List<KClass<*>>) {
             val code = classes.joinToString("\n") {

core/generated-sources/src/test/kotlin/org/jetbrains/kotlinx/dataframe/api/pivot.kt

@@ -153,4 +153,19 @@ class PivotTests {
                 1, -1, 5
             )
     }
+
+    @Test
+    fun `pivot then in aggregate`() {
+        val df = dataFrameOf(
+            "category1" to List(12) { it % 3 },
+            "category2" to List(12) { "category2_${it % 2}" },
+            "category3" to List(12) { "category3_${it % 5}" },
+            "value" to List(12) { it }
+        )
+
+        val df1 = df.groupBy("category1").aggregate {
+            pivot { "category2" then "category3" }.count()
+        }
+        df1 shouldBe df.pivot { "category2" then "category3" }.groupBy("category1").count()
+    }
 }

core/generated-sources/src/test/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/JupyterCodegenTests.kt

@@ -301,6 +301,23 @@ class JupyterCodegenTests : JupyterReplTestCase() {
         res2.shouldBeInstanceOf<Unit>()
     }
 
+    @Test
+    fun `type converter does not conflict with other type converters`() {
+        @Language("kts")
+        val anotherTypeConverter = """
+            notebook.fieldsHandlersProcessor.register(
+                FieldHandlerFactory.createUpdateHandler<ByteArray>(TypeDetection.RUNTIME) { _, prop ->
+                     execute(prop.name + ".toList()").name
+                },
+                ProcessingPriority.LOW
+            )
+        """.trimIndent()
+        execEx(anotherTypeConverter)
+        execEx("val x = ByteArray(1)")
+        val res1 = execRaw("x")
+        res1.shouldBeInstanceOf<List<*>>()
+    }
+
     @Test
     fun `generate a new marker when dataframe marker is not a data schema so that columns are accessible with extensions`() {
         @Language("kts")

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/pivot.kt

@@ -148,7 +148,7 @@ public fun <G> GroupBy<*, G>.pivotCounts(vararg columns: KProperty<*>, inward: B
 
 // region pivot
 
-public fun <T> AggregateGroupedDsl<T>.pivot(inward: Boolean = true, columns: ColumnsSelector<T, *>): PivotGroupBy<T> =
+public fun <T> AggregateGroupedDsl<T>.pivot(inward: Boolean = true, columns: PivotColumnsSelector<T, *>): PivotGroupBy<T> =
     PivotInAggregateImpl(this, columns, inward)
 
 public fun <T> AggregateGroupedDsl<T>.pivot(vararg columns: String, inward: Boolean = true): PivotGroupBy<T> =

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/shuffle.kt

@@ -3,15 +3,20 @@ package org.jetbrains.kotlinx.dataframe.api
 import org.jetbrains.kotlinx.dataframe.DataColumn
 import org.jetbrains.kotlinx.dataframe.DataFrame
 import org.jetbrains.kotlinx.dataframe.indices
+import kotlin.random.Random
 
 // region DataColumn
 
+public fun <T> DataColumn<T>.shuffle(random: Random): DataColumn<T> = get(indices.shuffled(random))
+
 public fun <T> DataColumn<T>.shuffle(): DataColumn<T> = get(indices.shuffled())
 
 // endregion
 
 // region DataFrame
 
+public fun <T> DataFrame<T>.shuffle(random: Random): DataFrame<T> = getRows(indices.shuffled(random))
+
 public fun <T> DataFrame<T>.shuffle(): DataFrame<T> = getRows(indices.shuffled())
 
 // endregion