From 404253853586df72426eeadd2f5b198983f2ee8d Mon Sep 17 00:00:00 2001 From: Steve Suh Date: Mon, 8 Mar 2021 23:41:44 -0800 Subject: [PATCH 1/6] initial commit --- azure-pipelines.yml | 58 +++++++++++++++++++ .../DeltaTableTests.cs | 4 +- .../IpcTests/Sql/DataFrameWriterTests.cs | 13 +++-- .../Sql/Streaming/DataStreamReaderTests.cs | 2 +- .../Processor/CommandProcessor.cs | 1 + .../Processor/TaskContextProcessor.cs | 1 + src/csharp/Microsoft.Spark/Broadcast.cs | 1 + src/csharp/Microsoft.Spark/Sql/DataFrame.cs | 1 + src/csharp/Microsoft.Spark/Versions.cs | 1 + src/scala/microsoft-spark-3-0/pom.xml | 2 +- src/scala/pom.xml | 1 + 11 files changed, 75 insertions(+), 10 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 38481b25c..82a31058a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -44,6 +44,54 @@ variables: forwardCompatibleTestOptions_Windows_3_0_2: "--filter FullyQualifiedName=NONE" forwardCompatibleTestOptions_Linux_3_0_2: $(forwardCompatibleTestOptions_Windows_3_0_2) + # Skip backwardCompatible tests because Microsoft.Spark.Worker requires Spark 3.1 support in + # CommandProcessor.cs and TaskContextProcessor.cs. Support added in https://github.com/dotnet/spark/pull/835 + backwardCompatibleTestOptions_Windows_3_1: "--filter \ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestDataFrameGroupedMapUdf)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestGroupedMapUdf&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfComplexTypesTests.TestUdfRegistrationWithReturnAsRowType)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfComplexTypesTests.TestUdfWithArrayChain)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfComplexTypesTests.TestUdfWithSimpleArrayType)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfComplexTypesTests.TestUdfWithMapType)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfComplexTypesTests.TestUdfWithRowArrayType)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfComplexTypesTests.TestUdfWithReturnAsMapType)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfComplexTypesTests.TestUdfWithReturnAsArrayOfArrayType)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfComplexTypesTests.TestUdfWithArrayOfArrayType)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfComplexTypesTests.TestUdfWithMapOfMapType)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfComplexTypesTests.TestUdfWithReturnAsSimpleArrayType)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfComplexTypesTests.TestUdfWithRowType)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfComplexTypesTests.TestUdfWithReturnAsRowType)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSerDeTests.TestExternalStaticMethodCall)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSerDeTests.TestInitExternalClassInUdf)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSerDeTests.TestUdfClosure)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithReturnAsDateType)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithReturnAsTimestampType)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithDateType)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.UdfTests.UdfSimpleTypesTests.TestUdfWithTimestampType)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.BroadcastTests.TestMultipleBroadcast)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.BroadcastTests.TestUnpersist)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.BroadcastTests.TestDestroy)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.PairRDDFunctionsTests.TestCollect)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.RDDTests.TestPipelinedRDD)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.RDDTests.TestMap)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.RDDTests.TestFlatMap)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.RDDTests.TestMapPartitions)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.RDDTests.TestMapPartitionsWithIndex)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.RDDTests.TestTextFile)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.RDDTests.TestFilter)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestDataFrameVectorUdf)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestVectorUdf)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestWithColumn)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestUDF)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.SparkSessionExtensionsTests.TestVersion)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataStreamWriterTests.TestForeachBatch)&\ + (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataStreamWriterTests.TestForeach)" + # Skip all forwardCompatible tests since microsoft-spark-3-1 jar does not get built when + # building forwardCompatible repo. + forwardCompatibleTestOptions_Windows_3_1: "--filter FullyQualifiedName=NONE" + backwardCompatibleTestOptions_Linux_3_1: $(backwardCompatibleTestOptions_Windows_3_1) + forwardCompatibleTestOptions_Linux_3_1: $(forwardCompatibleTestOptions_Windows_3_1) + # Azure DevOps variables are transformed into environment variables, with these variables we # avoid the first time experience and telemetry to speed up the build. DOTNET_CLI_TELEMETRY_OPTOUT: 1 @@ -361,3 +409,13 @@ stages: testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_0) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_0_2) + - version: '3.1.1' + jobOptions: + - pool: 'Hosted VS2017' + testOptions: "" + backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_1) + forwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_1) + - pool: 'Hosted Ubuntu 1604' + testOptions: "" + backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_1) + forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_1) diff --git a/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaTableTests.cs b/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaTableTests.cs index 75ac4705d..b72cba679 100644 --- a/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaTableTests.cs +++ b/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaTableTests.cs @@ -30,7 +30,7 @@ public DeltaTableTests(DeltaFixture fixture) /// Run the end-to-end scenario from the Delta Quickstart tutorial. /// /// - [SkipIfSparkVersionIsLessThan(Versions.V2_4_2)] + [SkipIfSparkVersionIsNotInRange(Versions.V2_4_2, Versions.V3_1_1)] public void TestTutorialScenario() { using var tempDirectory = new TemporaryDirectory(); @@ -223,7 +223,7 @@ void testWrapper( /// /// Test that methods return the expected signature. /// - [SkipIfSparkVersionIsLessThan(Versions.V2_4_2)] + [SkipIfSparkVersionIsNotInRange(Versions.V2_4_2, Versions.V3_1_1)] public void TestSignaturesV2_4_X() { using var tempDirectory = new TemporaryDirectory(); diff --git a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameWriterTests.cs b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameWriterTests.cs index 4f0d06742..f974d2b3d 100644 --- a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameWriterTests.cs +++ b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameWriterTests.cs @@ -69,12 +69,7 @@ public void TestSignaturesV2_3_X() // TODO: Test dfw.Jdbc without running a local db. - dfw.Option("path", tempDir.Path).SaveAsTable("TestTable"); - - dfw.InsertInto("TestTable"); - - dfw.Option("path", $"{tempDir.Path}TestSavePath1").Save(); - dfw.Save($"{tempDir.Path}TestSavePath2"); + dfw.Save($"{tempDir.Path}TestSavePath1"); dfw.Json($"{tempDir.Path}TestJsonPath"); @@ -85,6 +80,12 @@ public void TestSignaturesV2_3_X() dfw.Text($"{tempDir.Path}TestTextPath"); dfw.Csv($"{tempDir.Path}TestCsvPath"); + + dfw.Option("path", tempDir.Path).SaveAsTable("TestTable"); + + dfw.InsertInto("TestTable"); + + dfw.Option("path", $"{tempDir.Path}TestSavePath2").Save(); } } } diff --git a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/Streaming/DataStreamReaderTests.cs b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/Streaming/DataStreamReaderTests.cs index 9cf92984e..e777069af 100644 --- a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/Streaming/DataStreamReaderTests.cs +++ b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/Streaming/DataStreamReaderTests.cs @@ -52,7 +52,6 @@ public void TestSignaturesV2_3_X() })); string jsonFilePath = Path.Combine(TestEnvironment.ResourceDirectory, "people.json"); - Assert.IsType(dsr.Format("json").Option("path", jsonFilePath).Load()); Assert.IsType(dsr.Format("json").Load(jsonFilePath)); Assert.IsType(dsr.Json(jsonFilePath)); Assert.IsType( @@ -63,6 +62,7 @@ public void TestSignaturesV2_3_X() dsr.Parquet(Path.Combine(TestEnvironment.ResourceDirectory, "users.parquet"))); Assert.IsType (dsr.Text(Path.Combine(TestEnvironment.ResourceDirectory, "people.txt"))); + Assert.IsType(dsr.Format("json").Option("path", jsonFilePath).Load()); } } } diff --git a/src/csharp/Microsoft.Spark.Worker/Processor/CommandProcessor.cs b/src/csharp/Microsoft.Spark.Worker/Processor/CommandProcessor.cs index c85fff988..2c9badbb8 100644 --- a/src/csharp/Microsoft.Spark.Worker/Processor/CommandProcessor.cs +++ b/src/csharp/Microsoft.Spark.Worker/Processor/CommandProcessor.cs @@ -100,6 +100,7 @@ private static SqlCommand[] ReadSqlCommands( (2, 3) => SqlCommandProcessorV2_3_X.Process(evalType, stream), (2, 4) => SqlCommandProcessorV2_4_X.Process(evalType, stream), (3, 0) => SqlCommandProcessorV2_4_X.Process(evalType, stream), + (3, 1) => SqlCommandProcessorV2_4_X.Process(evalType, stream), _ => throw new NotSupportedException($"Spark {version} not supported.") }; } diff --git a/src/csharp/Microsoft.Spark.Worker/Processor/TaskContextProcessor.cs b/src/csharp/Microsoft.Spark.Worker/Processor/TaskContextProcessor.cs index 334a4d25c..6e96e1bad 100644 --- a/src/csharp/Microsoft.Spark.Worker/Processor/TaskContextProcessor.cs +++ b/src/csharp/Microsoft.Spark.Worker/Processor/TaskContextProcessor.cs @@ -24,6 +24,7 @@ internal TaskContext Process(Stream stream) (2, 3) => TaskContextProcessorV2_3_X.Process(stream), (2, 4) => TaskContextProcessorV2_4_X.Process(stream), (3, 0) => TaskContextProcessorV3_0_X.Process(stream), + (3, 1) => TaskContextProcessorV3_0_X.Process(stream), _ => throw new NotSupportedException($"Spark {_version} not supported.") }; } diff --git a/src/csharp/Microsoft.Spark/Broadcast.cs b/src/csharp/Microsoft.Spark/Broadcast.cs index 8f833dbb0..e944ae1f4 100644 --- a/src/csharp/Microsoft.Spark/Broadcast.cs +++ b/src/csharp/Microsoft.Spark/Broadcast.cs @@ -130,6 +130,7 @@ private JvmObjectReference CreateBroadcast(SparkContext sc, T value) (2, 3) => CreateBroadcast_V2_3_2_AndAbove(javaSparkContext, sc, value), (2, 4) => CreateBroadcast_V2_3_2_AndAbove(javaSparkContext, sc, value), (3, 0) => CreateBroadcast_V2_3_2_AndAbove(javaSparkContext, sc, value), + (3, 1) => CreateBroadcast_V2_3_2_AndAbove(javaSparkContext, sc, value), _ => throw new NotSupportedException($"Spark {version} not supported.") }; } diff --git a/src/csharp/Microsoft.Spark/Sql/DataFrame.cs b/src/csharp/Microsoft.Spark/Sql/DataFrame.cs index 80559b8a9..695b846b6 100644 --- a/src/csharp/Microsoft.Spark/Sql/DataFrame.cs +++ b/src/csharp/Microsoft.Spark/Sql/DataFrame.cs @@ -1058,6 +1058,7 @@ private IEnumerable GetRows(string funcName, params object[] args) (2, 3, _) => ParseConnectionInfo(result, false), (2, 4, _) => ParseConnectionInfo(result, false), (3, 0, _) => ParseConnectionInfo(result, false), + (3, 1, _) => ParseConnectionInfo(result, false), _ => throw new NotSupportedException($"Spark {version} not supported.") }; } diff --git a/src/csharp/Microsoft.Spark/Versions.cs b/src/csharp/Microsoft.Spark/Versions.cs index 893201f84..2822685e2 100644 --- a/src/csharp/Microsoft.Spark/Versions.cs +++ b/src/csharp/Microsoft.Spark/Versions.cs @@ -13,5 +13,6 @@ internal static class Versions internal const string V2_4_0 = "2.4.0"; internal const string V2_4_2 = "2.4.2"; internal const string V3_0_0 = "3.0.0"; + internal const string V3_1_1 = "3.1.1"; } } diff --git a/src/scala/microsoft-spark-3-0/pom.xml b/src/scala/microsoft-spark-3-0/pom.xml index 9ece1794f..90294ab38 100644 --- a/src/scala/microsoft-spark-3-0/pom.xml +++ b/src/scala/microsoft-spark-3-0/pom.xml @@ -10,7 +10,7 @@ 2019 UTF-8 - 2.12.8 + 2.12.10 2.12 3.0.0 diff --git a/src/scala/pom.xml b/src/scala/pom.xml index bb0b408ae..acd743343 100644 --- a/src/scala/pom.xml +++ b/src/scala/pom.xml @@ -14,6 +14,7 @@ microsoft-spark-2-3 microsoft-spark-2-4 microsoft-spark-3-0 + microsoft-spark-3-1 From 91be8f4ceaadd06f1e2c403ee46cb307732f5bd5 Mon Sep 17 00:00:00 2001 From: Steve Suh Date: Mon, 8 Mar 2021 23:50:43 -0800 Subject: [PATCH 2/6] Update azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 82a31058a..531a700b0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -45,7 +45,7 @@ variables: forwardCompatibleTestOptions_Linux_3_0_2: $(forwardCompatibleTestOptions_Windows_3_0_2) # Skip backwardCompatible tests because Microsoft.Spark.Worker requires Spark 3.1 support in - # CommandProcessor.cs and TaskContextProcessor.cs. Support added in https://github.com/dotnet/spark/pull/835 + # CommandProcessor.cs and TaskContextProcessor.cs. Support added in https://github.com/dotnet/spark/pull/836 backwardCompatibleTestOptions_Windows_3_1: "--filter \ (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestDataFrameGroupedMapUdf)&\ (FullyQualifiedName!=Microsoft.Spark.E2ETest.IpcTests.DataFrameTests.TestGroupedMapUdf&\ From ca930cf4a073158e08793ebf890491c462496ab8 Mon Sep 17 00:00:00 2001 From: Steve Suh Date: Mon, 15 Mar 2021 20:24:36 -0700 Subject: [PATCH 3/6] adjust pattern match --- .../Microsoft.Spark.Worker/Processor/CommandProcessor.cs | 3 +-- .../Microsoft.Spark.Worker/Processor/TaskContextProcessor.cs | 3 +-- src/csharp/Microsoft.Spark/Broadcast.cs | 3 +-- src/csharp/Microsoft.Spark/Sql/DataFrame.cs | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/csharp/Microsoft.Spark.Worker/Processor/CommandProcessor.cs b/src/csharp/Microsoft.Spark.Worker/Processor/CommandProcessor.cs index 2c9badbb8..1619ea6aa 100644 --- a/src/csharp/Microsoft.Spark.Worker/Processor/CommandProcessor.cs +++ b/src/csharp/Microsoft.Spark.Worker/Processor/CommandProcessor.cs @@ -99,8 +99,7 @@ private static SqlCommand[] ReadSqlCommands( { (2, 3) => SqlCommandProcessorV2_3_X.Process(evalType, stream), (2, 4) => SqlCommandProcessorV2_4_X.Process(evalType, stream), - (3, 0) => SqlCommandProcessorV2_4_X.Process(evalType, stream), - (3, 1) => SqlCommandProcessorV2_4_X.Process(evalType, stream), + (3, _) => SqlCommandProcessorV2_4_X.Process(evalType, stream), _ => throw new NotSupportedException($"Spark {version} not supported.") }; } diff --git a/src/csharp/Microsoft.Spark.Worker/Processor/TaskContextProcessor.cs b/src/csharp/Microsoft.Spark.Worker/Processor/TaskContextProcessor.cs index 6e96e1bad..45da43f2c 100644 --- a/src/csharp/Microsoft.Spark.Worker/Processor/TaskContextProcessor.cs +++ b/src/csharp/Microsoft.Spark.Worker/Processor/TaskContextProcessor.cs @@ -23,8 +23,7 @@ internal TaskContext Process(Stream stream) { (2, 3) => TaskContextProcessorV2_3_X.Process(stream), (2, 4) => TaskContextProcessorV2_4_X.Process(stream), - (3, 0) => TaskContextProcessorV3_0_X.Process(stream), - (3, 1) => TaskContextProcessorV3_0_X.Process(stream), + (3, _) => TaskContextProcessorV3_0_X.Process(stream), _ => throw new NotSupportedException($"Spark {_version} not supported.") }; } diff --git a/src/csharp/Microsoft.Spark/Broadcast.cs b/src/csharp/Microsoft.Spark/Broadcast.cs index e944ae1f4..f6e2697b0 100644 --- a/src/csharp/Microsoft.Spark/Broadcast.cs +++ b/src/csharp/Microsoft.Spark/Broadcast.cs @@ -129,8 +129,7 @@ private JvmObjectReference CreateBroadcast(SparkContext sc, T value) CreateBroadcast_V2_3_1_AndBelow(javaSparkContext, value), (2, 3) => CreateBroadcast_V2_3_2_AndAbove(javaSparkContext, sc, value), (2, 4) => CreateBroadcast_V2_3_2_AndAbove(javaSparkContext, sc, value), - (3, 0) => CreateBroadcast_V2_3_2_AndAbove(javaSparkContext, sc, value), - (3, 1) => CreateBroadcast_V2_3_2_AndAbove(javaSparkContext, sc, value), + (3, _) => CreateBroadcast_V2_3_2_AndAbove(javaSparkContext, sc, value), _ => throw new NotSupportedException($"Spark {version} not supported.") }; } diff --git a/src/csharp/Microsoft.Spark/Sql/DataFrame.cs b/src/csharp/Microsoft.Spark/Sql/DataFrame.cs index 695b846b6..e9da9fbdf 100644 --- a/src/csharp/Microsoft.Spark/Sql/DataFrame.cs +++ b/src/csharp/Microsoft.Spark/Sql/DataFrame.cs @@ -1057,8 +1057,7 @@ private IEnumerable GetRows(string funcName, params object[] args) // string to use for the authentication. (2, 3, _) => ParseConnectionInfo(result, false), (2, 4, _) => ParseConnectionInfo(result, false), - (3, 0, _) => ParseConnectionInfo(result, false), - (3, 1, _) => ParseConnectionInfo(result, false), + (3, _, _) => ParseConnectionInfo(result, false), _ => throw new NotSupportedException($"Spark {version} not supported.") }; } From 9ed9061aa07a7607c2efdacfe6bebb7fa67d60bc Mon Sep 17 00:00:00 2001 From: Steve Suh Date: Mon, 15 Mar 2021 21:14:13 -0700 Subject: [PATCH 4/6] add comments to code --- .../DeltaTableTests.cs | 8 ++++++++ .../IpcTests/Sql/DataFrameWriterTests.cs | 4 ++++ .../IpcTests/Sql/Streaming/DataStreamReaderTests.cs | 5 +++++ 3 files changed, 17 insertions(+) diff --git a/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaTableTests.cs b/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaTableTests.cs index b72cba679..f997e95e3 100644 --- a/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaTableTests.cs +++ b/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaTableTests.cs @@ -30,6 +30,10 @@ public DeltaTableTests(DeltaFixture fixture) /// Run the end-to-end scenario from the Delta Quickstart tutorial. /// /// + /// + /// Delta 0.8.0 is not compatible with Spark 3.1.1 + /// Disable Delta tests that have code path that creates a + /// `org.apache.spark.sql.catalyst.expressions.Alias` object. [SkipIfSparkVersionIsNotInRange(Versions.V2_4_2, Versions.V3_1_1)] public void TestTutorialScenario() { @@ -223,6 +227,10 @@ void testWrapper( /// /// Test that methods return the expected signature. /// + /// + /// Delta 0.8.0 is not compatible with Spark 3.1.1 + /// Disable Delta tests that have code path that creates a + /// `org.apache.spark.sql.catalyst.expressions.Alias` object. [SkipIfSparkVersionIsNotInRange(Versions.V2_4_2, Versions.V3_1_1)] public void TestSignaturesV2_4_X() { diff --git a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameWriterTests.cs b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameWriterTests.cs index f974d2b3d..2145367cb 100644 --- a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameWriterTests.cs +++ b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameWriterTests.cs @@ -85,6 +85,10 @@ public void TestSignaturesV2_3_X() dfw.InsertInto("TestTable"); + // In Spark 3.1.1+ setting path Option and then calling .Save(path) is not + // supported by unless `spark.sql.legacy.pathOptionBehavior.enabled` conf is set. + // .Json(path), .Parquet(path), etc calls .Save(path) under the hood so the conf + // needs to be set in these scenarios as well. dfw.Option("path", $"{tempDir.Path}TestSavePath2").Save(); } } diff --git a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/Streaming/DataStreamReaderTests.cs b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/Streaming/DataStreamReaderTests.cs index e777069af..dcceca931 100644 --- a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/Streaming/DataStreamReaderTests.cs +++ b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/Streaming/DataStreamReaderTests.cs @@ -62,6 +62,11 @@ public void TestSignaturesV2_3_X() dsr.Parquet(Path.Combine(TestEnvironment.ResourceDirectory, "users.parquet"))); Assert.IsType (dsr.Text(Path.Combine(TestEnvironment.ResourceDirectory, "people.txt"))); + + // In Spark 3.1.1+ setting the `path` Option and then calling .Load(path) is not + // supported by unless `spark.sql.legacy.pathOptionBehavior.enabled` conf is set. + // .Json(path), .Parquet(path), etc follow the same code path so the conf + // needs to be set in these scenarios as well. Assert.IsType(dsr.Format("json").Option("path", jsonFilePath).Load()); } } From ad795c2e9aa8b7de4de67d08fc97afdeedbd8859 Mon Sep 17 00:00:00 2001 From: Steve Suh Date: Tue, 16 Mar 2021 09:17:45 -0700 Subject: [PATCH 5/6] add comments to code --- .../IpcTests/Sql/DataFrameWriterTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameWriterTests.cs b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameWriterTests.cs index 2145367cb..f6dfd8755 100644 --- a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameWriterTests.cs +++ b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameWriterTests.cs @@ -85,9 +85,9 @@ public void TestSignaturesV2_3_X() dfw.InsertInto("TestTable"); - // In Spark 3.1.1+ setting path Option and then calling .Save(path) is not + // In Spark 3.1.1+ setting the `path` Option and then calling .Save(path) is not // supported by unless `spark.sql.legacy.pathOptionBehavior.enabled` conf is set. - // .Json(path), .Parquet(path), etc calls .Save(path) under the hood so the conf + // .Json(path), .Parquet(path), etc follow the same code path so the conf // needs to be set in these scenarios as well. dfw.Option("path", $"{tempDir.Path}TestSavePath2").Save(); } From 2886ffd6181a13b59149ce7124e5e7344ccdb74e Mon Sep 17 00:00:00 2001 From: Steve Suh Date: Tue, 16 Mar 2021 11:44:00 -0700 Subject: [PATCH 6/6] update comment --- .../DeltaTableTests.cs | 4 ++-- .../IpcTests/Sql/DataFrameWriterTests.cs | 2 +- .../IpcTests/Sql/Streaming/DataStreamReaderTests.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaTableTests.cs b/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaTableTests.cs index f997e95e3..bd15ad3e6 100644 --- a/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaTableTests.cs +++ b/src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaTableTests.cs @@ -32,7 +32,7 @@ public DeltaTableTests(DeltaFixture fixture) /// /// /// Delta 0.8.0 is not compatible with Spark 3.1.1 - /// Disable Delta tests that have code path that creates a + /// Disable Delta tests that have code paths that create an /// `org.apache.spark.sql.catalyst.expressions.Alias` object. [SkipIfSparkVersionIsNotInRange(Versions.V2_4_2, Versions.V3_1_1)] public void TestTutorialScenario() @@ -229,7 +229,7 @@ void testWrapper( /// /// /// Delta 0.8.0 is not compatible with Spark 3.1.1 - /// Disable Delta tests that have code path that creates a + /// Disable Delta tests that have code paths that create an /// `org.apache.spark.sql.catalyst.expressions.Alias` object. [SkipIfSparkVersionIsNotInRange(Versions.V2_4_2, Versions.V3_1_1)] public void TestSignaturesV2_4_X() diff --git a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameWriterTests.cs b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameWriterTests.cs index f6dfd8755..c693fdabc 100644 --- a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameWriterTests.cs +++ b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameWriterTests.cs @@ -86,7 +86,7 @@ public void TestSignaturesV2_3_X() dfw.InsertInto("TestTable"); // In Spark 3.1.1+ setting the `path` Option and then calling .Save(path) is not - // supported by unless `spark.sql.legacy.pathOptionBehavior.enabled` conf is set. + // supported unless `spark.sql.legacy.pathOptionBehavior.enabled` conf is set. // .Json(path), .Parquet(path), etc follow the same code path so the conf // needs to be set in these scenarios as well. dfw.Option("path", $"{tempDir.Path}TestSavePath2").Save(); diff --git a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/Streaming/DataStreamReaderTests.cs b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/Streaming/DataStreamReaderTests.cs index dcceca931..f7f8cec24 100644 --- a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/Streaming/DataStreamReaderTests.cs +++ b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/Streaming/DataStreamReaderTests.cs @@ -64,7 +64,7 @@ public void TestSignaturesV2_3_X() (dsr.Text(Path.Combine(TestEnvironment.ResourceDirectory, "people.txt"))); // In Spark 3.1.1+ setting the `path` Option and then calling .Load(path) is not - // supported by unless `spark.sql.legacy.pathOptionBehavior.enabled` conf is set. + // supported unless `spark.sql.legacy.pathOptionBehavior.enabled` conf is set. // .Json(path), .Parquet(path), etc follow the same code path so the conf // needs to be set in these scenarios as well. Assert.IsType(dsr.Format("json").Option("path", jsonFilePath).Load());