typelevel#787 - reduce the case class api usage even further and Crea…

…teStruct, and allow shims for the deprecated functions
chris-twiner · Mar 7, 2024 · 4933a90 · 4933a90
1 parent 6717e4b
commit 4933a90
Show file tree

Hide file tree

Showing 9 changed files with 259 additions and 203 deletions.
diff --git a/build.sbt b/build.sbt
@@ -17,7 +17,7 @@ val shimVersion = "0.0.1-SNAPSHOT"
 val Scala212 = "2.12.19"
 val Scala213 = "2.13.13"
 
-//resolvers in Global += Resolver.mavenLocal
+resolvers in Global += Resolver.mavenLocal
 resolvers in Global += MavenRepository(
   "sonatype-s01-snapshots",
   Resolver.SonatypeS01RepositoryRoot + "/snapshots"

diff --git a/dataset/src/main/scala/frameless/RecordEncoder.scala b/dataset/src/main/scala/frameless/RecordEncoder.scala
@@ -8,11 +8,7 @@ import com.sparkutils.shim.expressions.{
 }
 import com.sparkutils.shim.{ deriveUnitLiteral, ifIsNull }
 import org.apache.spark.sql.FramelessInternals
-import org.apache.spark.sql.catalyst.expressions.{
-  CreateNamedStruct => _,
-  GetStructField => _,
-  _
-}
+import org.apache.spark.sql.catalyst.expressions.{ Expression, Literal }
 import org.apache.spark.sql.shim.{
   Invoke5 => Invoke,
   NewInstance4 => NewInstance

diff --git a/dataset/src/main/scala/frameless/TypedColumn.scala b/dataset/src/main/scala/frameless/TypedColumn.scala
@@ -5,13 +5,7 @@ import frameless.syntax._
 
 import org.apache.spark.sql.catalyst.expressions.{
   Expression,
-  EqualNullSafe,
-  EqualTo,
-  Not,
-  IsNull,
-  IsNotNull,
-  Literal,
-  Coalesce
+  Literal
 } // 787 - Spark 4 source code compat
 import org.apache.spark.sql.types.DecimalType
 import org.apache.spark.sql.{ Column, FramelessInternals }
@@ -22,6 +16,15 @@ import shapeless.ops.record.Selector
 import scala.annotation.implicitNotFound
 import scala.reflect.ClassTag
 
+import com.sparkutils.shim.expressions.{
+  EqualNullSafe2 => EqualNullSafe,
+  EqualTo2 => EqualTo,
+  Not1 => Not,
+  IsNull1 => IsNull,
+  IsNotNull1 => IsNotNull,
+  Coalesce1 => Coalesce
+} // 787 - Spark 4 source code compat
+
 import scala.language.experimental.macros
 
 sealed trait UntypedExpression[T] {

diff --git a/dataset/src/main/scala/frameless/TypedEncoder.scala b/dataset/src/main/scala/frameless/TypedEncoder.scala
@@ -9,7 +9,7 @@ import scala.reflect.ClassTag
 import org.apache.spark.sql.FramelessInternals
 import org.apache.spark.sql.FramelessInternals.UserDefinedType
 import org.apache.spark.sql.{ reflection => ScalaReflection }
-import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.{ Expression, UnsafeArrayData, Literal }
 import org.apache.spark.sql.catalyst.util.{
   ArrayBasedMapData,
   DateTimeUtils,