Skip to content

Commit 8e67b77

Browse files
alvarosantossyngentaalvarosantossyngenta
committed
feat: add compatibility with Spark 3.5 #2 
1 parent ab9abae commit 8e67b77

File tree

4 files changed

+51
-23
lines changed

4 files changed

+51
-23
lines changed

‎.devcontainer/docker-compose.yml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,17 @@ x-environment:
1111
MINIO_ROOT_USER: minio
1212
MINIO_ROOT_PASSWORD: CHANGEME123
1313
TEST_BUCKET_NAME: test-bucket
14+
JDK_JAVA_OPTIONS: |
15+
--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED
16+
--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
17+
--add-opens=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED
18+
--add-opens=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED
19+
--add-opens=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED
20+
--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED
21+
--add-opens=java.base/java.io=ALL-UNNAMED
22+
--add-opens=java.base/java.nio=ALL-UNNAMED
23+
--add-opens=java.base/java.util=ALL-UNNAMED
24+
--add-opens=java.base/java.lang=ALL-UNNAMED
1425
services:
1526
main:
1627
image: ghcr.io/syngenta-digital/service-java-data-pipelines-metorikku:devcontainer

‎build.sbt

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ val sparkShortVersion: Def.Initialize[String] = Def.setting {
2727
}
2828

2929
val jacksonVersion: Def.Initialize[String] = Def.setting {
30-
Option(System.getenv("JACKSON_VERSION")).getOrElse("2.12.7")
30+
Option(System.getenv("JACKSON_VERSION")).getOrElse("2.16.2")
3131
}
3232

3333
val sparkTestVersion: Def.Initialize[String] = Def.setting {
@@ -58,14 +58,14 @@ lazy val excludeJacksonModule = ExclusionRule(organization = "com.fasterxml.
5858
lazy val excludeAWS = ExclusionRule(organization = "com.amazonaws")
5959

6060
libraryDependencies ++= Seq(
61-
"org.apache.spark" %% "spark-core" % sparkVersion.value % "provided",
62-
"org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided",
63-
"org.apache.spark" %% "spark-mllib" % sparkVersion.value % "provided",
64-
"org.apache.spark" %% "spark-hive" % sparkVersion.value % "provided",
65-
"org.apache.spark" %% "spark-sql-kafka-0-10" % sparkVersion.value % "provided",
66-
"org.apache.spark" %% "spark-streaming" % sparkVersion.value % "provided",
67-
"org.apache.spark" %% "spark-avro" % sparkVersion.value % "provided",
68-
"org.apache.spark" %% "spark-hadoop-cloud" % sparkVersion.value % "provided" excludeAll (excludeAWS),
61+
"org.apache.spark" %% "spark-core" % sparkVersion.value % "provided" excludeAll (excludeJacksonCore, excludeJacksonDataFormat, excludeJacksonDataType, excludeJacksonModule),
62+
"org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided" excludeAll (excludeJacksonCore, excludeJacksonDataFormat, excludeJacksonDataType, excludeJacksonModule),
63+
"org.apache.spark" %% "spark-mllib" % sparkVersion.value % "provided" excludeAll (excludeJacksonCore, excludeJacksonDataFormat, excludeJacksonDataType, excludeJacksonModule),
64+
"org.apache.spark" %% "spark-hive" % sparkVersion.value % "provided" excludeAll (excludeJacksonCore, excludeJacksonDataFormat, excludeJacksonDataType, excludeJacksonModule),
65+
"org.apache.spark" %% "spark-sql-kafka-0-10" % sparkVersion.value % "provided" excludeAll (excludeJacksonCore, excludeJacksonDataFormat, excludeJacksonDataType, excludeJacksonModule),
66+
"org.apache.spark" %% "spark-streaming" % sparkVersion.value % "provided" excludeAll (excludeJacksonCore, excludeJacksonDataFormat, excludeJacksonDataType, excludeJacksonModule),
67+
"org.apache.spark" %% "spark-avro" % sparkVersion.value % "provided" excludeAll (excludeJacksonCore, excludeJacksonDataFormat, excludeJacksonDataType, excludeJacksonModule),
68+
"org.apache.spark" %% "spark-hadoop-cloud" % sparkVersion.value % "provided" excludeAll (excludeAWS) excludeAll (excludeJacksonCore, excludeJacksonDataFormat, excludeJacksonDataType, excludeJacksonModule),
6969
"com.holdenkarau" %% "spark-testing-base" % sparkTestVersion.value % "test" excludeAll (excludeSpark),
7070
"com.github.scopt" %% "scopt" % "4.1.0",
7171
"org.scala-lang" % "scala-library" % scalaVersion.value,
@@ -86,19 +86,19 @@ libraryDependencies ++= Seq(
8686
"com.redislabs" %% "spark-redis" % "3.1.0",
8787
"org.apache.kafka" %% "kafka" % "3.9.0",
8888
"za.co.absa" %% "abris" % "3.2.2" % "provided" excludeAll (excludeAvro, excludeSpark),
89-
"org.apache.hudi" %% "hudi-spark-bundle" % "0.10.0" % "provided",
90-
"org.apache.parquet" % "parquet-avro" % "1.15.0" % "provided",
89+
"org.apache.hudi" %% "hudi-spark-bundle" % "0.10.0" % "provided",
90+
"org.apache.parquet" % "parquet-avro" % "1.15.0" % "provided" excludeAll (excludeJacksonCore, excludeJacksonDataFormat, excludeJacksonDataType, excludeJacksonModule),
9191
"com.amazon.deequ" % "deequ" % ("2.0.9-spark-" + sparkShortVersion.value) excludeAll (excludeSpark, excludeScalanlp),
92-
"org.apache.avro" % "avro" % "1.12.0" % "provided",
93-
"com.databricks" %% "spark-xml" % "0.18.0",
94-
"com.outr" %% "hasher" % "1.2.2",
95-
"org.mongodb.spark" %% "mongo-spark-connector" % "10.4.1",
96-
"mysql" % "mysql-connector-java" % "8.0.33",
97-
"org.apache.logging.log4j" % "log4j-api" % "2.24.3" % "provided",
98-
"org.apache.logging.log4j" % "log4j-core" % "2.24.3" % "provided",
99-
"org.apache.logging.log4j" % "log4j-slf4j-impl" % "2.24.3" % "provided",
100-
"org.postgresql" % "postgresql" % "42.7.5",
101-
"io.delta" %% "delta-core" % "2.4.0",
92+
"org.apache.avro" % "avro" % "1.12.0" % "provided" excludeAll (excludeJacksonCore, excludeJacksonDataFormat, excludeJacksonDataType, excludeJacksonModule),
93+
"com.databricks" %% "spark-xml" % "0.18.0",
94+
"com.outr" %% "hasher" % "1.2.2",
95+
"org.mongodb.spark" %% "mongo-spark-connector" % "10.4.1",
96+
"mysql" % "mysql-connector-java" % "8.0.33",
97+
"org.apache.logging.log4j" % "log4j-api" % "2.24.3" % "provided" excludeAll (excludeJacksonCore, excludeJacksonDataFormat, excludeJacksonDataType, excludeJacksonModule),
98+
"org.apache.logging.log4j" % "log4j-core" % "2.24.3" % "provided" excludeAll (excludeJacksonCore, excludeJacksonDataFormat, excludeJacksonDataType, excludeJacksonModule),
99+
"org.apache.logging.log4j" % "log4j-slf4j-impl" % "2.24.3" % "provided" excludeAll (excludeJacksonCore, excludeJacksonDataFormat, excludeJacksonDataType, excludeJacksonModule),
100+
"org.postgresql" % "postgresql" % "42.7.5",
101+
"io.delta" %% "delta-core" % "2.4.0",
102102
"io.vertx" % "vertx-json-schema" % "4.5.12" excludeAll (excludeJacksonCore, excludeJacksonDataFormat, excludeJacksonDataType, excludeJacksonModule),
103103
"com.google.guava" % "guava" % "25.1-jre",
104104
"org.apache.sedona" %% ("sedona-spark-" + sparkShortVersion.value) % "1.6.1" excludeAll (excludeSpark),

‎scripts/test.sh

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,23 @@ rm -Rf $MAIN_DIR/spark-warehouse $MAIN_DIR/metastore_db
99

1010
TEST_NAME=$1
1111

12+
export JDK_JAVA_OPTIONS='
13+
--add-opens=java.base/java.lang=ALL-UNNAMED
14+
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
15+
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
16+
--add-opens=java.base/java.io=ALL-UNNAMED
17+
--add-opens=java.base/java.net=ALL-UNNAMED
18+
--add-opens=java.base/java.nio=ALL-UNNAMED
19+
--add-opens=java.base/java.util=ALL-UNNAMED
20+
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED
21+
--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
22+
--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
23+
--add-opens=java.base/sun.nio.cs=ALL-UNNAMED
24+
--add-opens=java.base/sun.security.action=ALL-UNNAMED
25+
--add-opens=java.base/sun.util.calendar=ALL-UNNAMED
26+
--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED
27+
'
28+
1229
if [[ -z "$TEST_NAME" ]]; then
1330
echo "################################"
1431
echo "Executing unit tests"

‎src/test/scala/com/yotpo/metorikku/code/steps/functions/test/UserDefinedFunctionsTests.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ class JsonObjectTest extends AnyFunSuite {
108108
json,
109109
"$.store",
110110
compressOut = true
111-
) === "H4sIAAAAAAAAAI2QQWvDMAyF/4rxOQlLQqDOdRuU0e6QFXYYOziukoi41nCcjlDy32eXQd3TBkKHJ/Q96V14SzTy+uPClXTQk114zS10YMEo4AmXsxvIevEVe9CsAZi86tBp8OKbXND0E6OOuQHYIxg3e0TCvyz69XqTiWpN7uAdKodkYvTzGfRi2Luc+yGGf5M9BvSWDM32Rs2LTIi/sS8Za651ID0imAh98Mfufunh8CZ84ec4tcaPH9JSVGkuQt/cfIv/+W7BnqRhe9Bn1Boi2z21C3tCNcZWVVWmRV7meVreBSfWz4S3qBYVdr0p6SvewjGKQoSE1x9nfazryAEAAA=="
111+
) === "H4sIAAAAAAAA/42QQWvDMAyF/4rxOQlLQqDOdRuU0e6QFXYYOziukoi41nCcjlDy32eXQd3TBkKHJ/Q96V14SzTy+uPClXTQk114zS10YMEo4AmXsxvIevEVe9CsAZi86tBp8OKbXND0E6OOuQHYIxg3e0TCvyz69XqTiWpN7uAdKodkYvTzGfRi2Luc+yGGf5M9BvSWDM32Rs2LTIi/sS8Za651ID0imAh98Mfufunh8CZ84ec4tcaPH9JSVGkuQt/cfIv/+W7BnqRhe9Bn1Boi2z21C3tCNcZWVVWmRV7meVreBSfWz4S3qBYVdr0p6SvewjGKQoSE1x9nfazryAEAAA=="
112112
)
113113
}
114114

@@ -130,7 +130,7 @@ class JsonObjectTest extends AnyFunSuite {
130130
"$.book[2].category",
131131
compressIn = true,
132132
compressOut = true
133-
) === "H4sIAAAAAAAAAFNKy0wuyczPUwIAIxYt6wkAAAA="
133+
) === "H4sIAAAAAAAA/1NKy0wuyczPUwIAIxYt6wkAAAA="
134134
)
135135
}
136136

0 commit comments

Comments
 (0)