feat: add compatibility with Spark 3.5 #3

Author: alvarosantossyngenta

.devcontainer/Dockerfile

@@ -36,7 +36,7 @@ ADD .devcontainer/hive-site.xml "/tmp/"
 RUN curl -s "https://get.sdkman.io?rcupdate=false" | bash
 
 ENV JAVA_VERSION="17.0.13-amzn"
-ENV SCALA_VERSION="2.12.18"
+ENV SCALA_VERSION="2.12.19"
 ENV SCALA_BINARY_VERSION="2.12"
 ENV SBT_VERSION="1.10.6"
 ENV SPARK_VERSION="3.5.2"

.devcontainer/docker-compose.yml

@@ -12,16 +12,20 @@ x-environment:
   MINIO_ROOT_PASSWORD: CHANGEME123
   TEST_BUCKET_NAME: test-bucket
   JDK_JAVA_OPTIONS: |
-    --add-opens=java.base/jdk.internal.misc=ALL-UNNAMED
-    --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
-    --add-opens=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED
-    --add-opens=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED
-    --add-opens=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED
-    --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED
+    --add-opens=java.base/java.lang=ALL-UNNAMED
+    --add-opens=java.base/java.lang.invoke=ALL-UNNAMED
+    --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
     --add-opens=java.base/java.io=ALL-UNNAMED
+    --add-opens=java.base/java.net=ALL-UNNAMED
     --add-opens=java.base/java.nio=ALL-UNNAMED
     --add-opens=java.base/java.util=ALL-UNNAMED
-    --add-opens=java.base/java.lang=ALL-UNNAMED
+    --add-opens=java.base/java.util.concurrent=ALL-UNNAMED
+    --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
+    --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
+    --add-opens=java.base/sun.nio.cs=ALL-UNNAMED
+    --add-opens=java.base/sun.security.action=ALL-UNNAMED
+    --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
+    --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED
 services:
   main:
     image: ghcr.io/syngenta-digital/service-java-data-pipelines-metorikku:devcontainer

scripts/test.sh

@@ -9,23 +9,6 @@ rm -Rf $MAIN_DIR/spark-warehouse $MAIN_DIR/metastore_db
 
 TEST_NAME=$1
 
-export JDK_JAVA_OPTIONS='
---add-opens=java.base/java.lang=ALL-UNNAMED
---add-opens=java.base/java.lang.invoke=ALL-UNNAMED
---add-opens=java.base/java.lang.reflect=ALL-UNNAMED
---add-opens=java.base/java.io=ALL-UNNAMED
---add-opens=java.base/java.net=ALL-UNNAMED
---add-opens=java.base/java.nio=ALL-UNNAMED
---add-opens=java.base/java.util=ALL-UNNAMED
---add-opens=java.base/java.util.concurrent=ALL-UNNAMED
---add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
---add-opens=java.base/sun.nio.ch=ALL-UNNAMED
---add-opens=java.base/sun.nio.cs=ALL-UNNAMED
---add-opens=java.base/sun.security.action=ALL-UNNAMED
---add-opens=java.base/sun.util.calendar=ALL-UNNAMED
---add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED
-'
-
 if [[ -z "$TEST_NAME" ]]; then
     echo "################################"
     echo "Executing unit tests"

src/test/scala/com/yotpo/metorikku/test/MetorikkuTest.scala

@@ -76,7 +76,7 @@ class MetorikkuTest extends AnyFunSuite with BeforeAndAfterAll {
         )
       )
     }
-    assert(thrown.getMessage.startsWith("Path does not exist"))
+    assert(thrown.getMessage.contains("Path does not exist"))
 
   }