From c404ef03fcc9e50bc88c36ddc02a1b9666ac64e3 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Wed, 15 May 2019 19:56:50 -0700 Subject: [PATCH 1/5] Initial commit --- azure-pipelines.yml | 11 +++++++++++ script/download-spark-distros.cmd | 1 + src/scala/microsoft-spark-2.4.x/pom.xml | 2 +- .../scala/org/apache/spark/deploy/DotnetRunner.scala | 2 +- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6ffdca9a8..f68c01eee 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -130,6 +130,17 @@ jobs: HADOOP_HOME: $(Build.BinariesDirectory)\hadoop DOTNET_WORKER_DIR: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker\netcoreapp2.1\win-x64 + - task: DotNetCoreCLI@2 + displayName: 'E2E tests for Spark 2.4.3' + inputs: + command: test + projects: '**/Microsoft.Spark.E2ETest/*.csproj' + arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura' + env: + SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.3-bin-hadoop2.7 + HADOOP_HOME: $(Build.BinariesDirectory)\hadoop + DOTNET_WORKER_DIR: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker\netcoreapp2.1\win-x64 + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: CopyFiles@2 displayName: Stage .NET artifacts diff --git a/script/download-spark-distros.cmd b/script/download-spark-distros.cmd index 45d0bd584..2a8b8dd71 100644 --- a/script/download-spark-distros.cmd +++ b/script/download-spark-distros.cmd @@ -19,5 +19,6 @@ curl -k -L -o spark-2.3.2.tgz https://archive.apache.org/dist/spark/spark-2.3.2/ curl -k -L -o spark-2.3.3.tgz https://archive.apache.org/dist/spark/spark-2.3.3/spark-2.3.3-bin-hadoop2.7.tgz && tar xzvf spark-2.3.3.tgz curl -k -L -o spark-2.4.0.tgz https://archive.apache.org/dist/spark/spark-2.4.0/spark-2.4.0-bin-hadoop2.7.tgz && tar xzvf spark-2.4.0.tgz curl -k -L -o spark-2.4.1.tgz https://archive.apache.org/dist/spark/spark-2.4.1/spark-2.4.1-bin-hadoop2.7.tgz && tar xzvf spark-2.4.1.tgz +curl -k -L -o spark-2.4.3.tgz https://archive.apache.org/dist/spark/spark-2.4.3/spark-2.4.3-bin-hadoop2.7.tgz && tar xzvf spark-2.4.3.tgz endlocal \ No newline at end of file diff --git a/src/scala/microsoft-spark-2.4.x/pom.xml b/src/scala/microsoft-spark-2.4.x/pom.xml index 0825ca1a8..33f02c964 100644 --- a/src/scala/microsoft-spark-2.4.x/pom.xml +++ b/src/scala/microsoft-spark-2.4.x/pom.xml @@ -12,7 +12,7 @@ UTF-8 2.11.8 2.11 - 2.4.1 + 2.4.3 diff --git a/src/scala/microsoft-spark-2.4.x/src/main/scala/org/apache/spark/deploy/DotnetRunner.scala b/src/scala/microsoft-spark-2.4.x/src/main/scala/org/apache/spark/deploy/DotnetRunner.scala index 666fa4dbd..4cc623f3f 100644 --- a/src/scala/microsoft-spark-2.4.x/src/main/scala/org/apache/spark/deploy/DotnetRunner.scala +++ b/src/scala/microsoft-spark-2.4.x/src/main/scala/org/apache/spark/deploy/DotnetRunner.scala @@ -33,7 +33,7 @@ import scala.util.Try */ object DotnetRunner extends Logging { private val DEBUG_PORT = 5567 - private val supportedSparkVersions = Set[String]("2.4.0", "2.4.1") + private val supportedSparkVersions = Set[String]("2.4.0", "2.4.1", "2.4.3") val SPARK_VERSION = DotnetUtils.normalizeSparkVersion(spark.SPARK_VERSION) From 40edc24a61d1e461a79ab35fe9777c84ec2b3b8c Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Tue, 21 May 2019 13:34:43 -0700 Subject: [PATCH 2/5] Update README.md --- README.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/README.md b/README.md index 9a014cc42..d9576333e 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,46 @@ - [Code of Conduct](#code-of-conduct) - [License](#license) +## Supported Apache Spark Versions + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Apache Spark.NET for Apache Spark Release
2.3.0v0.2.0
2.3.1
2.3.2
2.3.3
2.4.0
2.4.1
2.4.2Not supported
2.4.3master branch
+ ## Get Started These instructions will show you how to run a .NET for Apache Spark app using .NET Core. - [Windows Instructions](docs/getting-started/windows-instructions.md) From 90853372699488ba998774cf330ecb5080a33705 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Tue, 21 May 2019 13:36:02 -0700 Subject: [PATCH 3/5] Update release-0.2.md --- docs/release-notes/0.2/release-0.2.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/release-notes/0.2/release-0.2.md b/docs/release-notes/0.2/release-0.2.md index a03512eca..c2d02c478 100644 --- a/docs/release-notes/0.2/release-0.2.md +++ b/docs/release-notes/0.2/release-0.2.md @@ -38,10 +38,7 @@ The following table outlines the supported Spark versions along with the microso 2.4.0 - microsoft-spark-2.4.x-0.2.0.jar - - - 2.4.0 + microsoft-spark-2.4.x-0.2.0.jar 2.4.1 From 5ae449777871afd5de538319e9337b28fcbadce2 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Tue, 21 May 2019 13:38:32 -0700 Subject: [PATCH 4/5] Update README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d9576333e..1f1f4e732 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ ## Table of Contents +- [Supported Apache Spark](#supported-apache-spark) - [Get Started](#get-started) - [Build Status](#build-status) - [Building from Source](#building-from-source) @@ -23,13 +24,13 @@ - [Code of Conduct](#code-of-conduct) - [License](#license) -## Supported Apache Spark Versions +## Supported Apache Spark - + From bc3ec9989fd89973b24b7f2ec595b41a98051583 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Fri, 31 May 2019 10:08:27 -0700 Subject: [PATCH 5/5] Update README.md --- README.md | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 1f1f4e732..e00c60d45 100644 --- a/README.md +++ b/README.md @@ -35,17 +35,8 @@ - - - - - - - - - - - + +
Apache Spark.NET for Apache Spark Release.NET for Apache Spark
2.3.0v0.2.0
2.3.1
2.3.2
2.3.32.3.*v0.2.0
2.4.0