diff --git a/README.md b/README.md
index 9a014cc42..e00c60d45 100644
--- a/README.md
+++ b/README.md
@@ -12,6 +12,7 @@
## Table of Contents
+- [Supported Apache Spark](#supported-apache-spark)
- [Get Started](#get-started)
- [Build Status](#build-status)
- [Building from Source](#building-from-source)
@@ -23,6 +24,37 @@
- [Code of Conduct](#code-of-conduct)
- [License](#license)
+## Supported Apache Spark
+
+
+
+
+ Apache Spark |
+ .NET for Apache Spark |
+
+
+
+
+ 2.3.* |
+ v0.2.0 |
+
+
+ 2.4.0 |
+
+
+ 2.4.1 |
+
+
+ 2.4.2 |
+ Not supported |
+
+
+ 2.4.3 |
+ master branch |
+
+
+
+
## Get Started
These instructions will show you how to run a .NET for Apache Spark app using .NET Core.
- [Windows Instructions](docs/getting-started/windows-instructions.md)
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index c4338852c..8622803fa 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -123,6 +123,17 @@ jobs:
HADOOP_HOME: $(Build.BinariesDirectory)\hadoop
DOTNET_WORKER_DIR: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker\netcoreapp2.1\win-x64
+ - task: DotNetCoreCLI@2
+ displayName: 'E2E tests for Spark 2.4.3'
+ inputs:
+ command: test
+ projects: '**/Microsoft.Spark.E2ETest/*.csproj'
+ arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
+ env:
+ SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.3-bin-hadoop2.7
+ HADOOP_HOME: $(Build.BinariesDirectory)\hadoop
+ DOTNET_WORKER_DIR: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker\netcoreapp2.1\win-x64
+
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: CopyFiles@2
displayName: Stage .NET artifacts
diff --git a/docs/release-notes/0.2/release-0.2.md b/docs/release-notes/0.2/release-0.2.md
index a03512eca..c2d02c478 100644
--- a/docs/release-notes/0.2/release-0.2.md
+++ b/docs/release-notes/0.2/release-0.2.md
@@ -38,10 +38,7 @@ The following table outlines the supported Spark versions along with the microso
2.4.0 |
- microsoft-spark-2.4.x-0.2.0.jar |
-
-
- 2.4.0 |
+ microsoft-spark-2.4.x-0.2.0.jar |
2.4.1 |
diff --git a/script/download-spark-distros.cmd b/script/download-spark-distros.cmd
index 45d0bd584..2a8b8dd71 100644
--- a/script/download-spark-distros.cmd
+++ b/script/download-spark-distros.cmd
@@ -19,5 +19,6 @@ curl -k -L -o spark-2.3.2.tgz https://archive.apache.org/dist/spark/spark-2.3.2/
curl -k -L -o spark-2.3.3.tgz https://archive.apache.org/dist/spark/spark-2.3.3/spark-2.3.3-bin-hadoop2.7.tgz && tar xzvf spark-2.3.3.tgz
curl -k -L -o spark-2.4.0.tgz https://archive.apache.org/dist/spark/spark-2.4.0/spark-2.4.0-bin-hadoop2.7.tgz && tar xzvf spark-2.4.0.tgz
curl -k -L -o spark-2.4.1.tgz https://archive.apache.org/dist/spark/spark-2.4.1/spark-2.4.1-bin-hadoop2.7.tgz && tar xzvf spark-2.4.1.tgz
+curl -k -L -o spark-2.4.3.tgz https://archive.apache.org/dist/spark/spark-2.4.3/spark-2.4.3-bin-hadoop2.7.tgz && tar xzvf spark-2.4.3.tgz
endlocal
\ No newline at end of file
diff --git a/src/scala/microsoft-spark-2.4.x/pom.xml b/src/scala/microsoft-spark-2.4.x/pom.xml
index 0825ca1a8..33f02c964 100644
--- a/src/scala/microsoft-spark-2.4.x/pom.xml
+++ b/src/scala/microsoft-spark-2.4.x/pom.xml
@@ -12,7 +12,7 @@
UTF-8
2.11.8
2.11
- 2.4.1
+ 2.4.3
diff --git a/src/scala/microsoft-spark-2.4.x/src/main/scala/org/apache/spark/deploy/DotnetRunner.scala b/src/scala/microsoft-spark-2.4.x/src/main/scala/org/apache/spark/deploy/DotnetRunner.scala
index 666fa4dbd..4cc623f3f 100644
--- a/src/scala/microsoft-spark-2.4.x/src/main/scala/org/apache/spark/deploy/DotnetRunner.scala
+++ b/src/scala/microsoft-spark-2.4.x/src/main/scala/org/apache/spark/deploy/DotnetRunner.scala
@@ -33,7 +33,7 @@ import scala.util.Try
*/
object DotnetRunner extends Logging {
private val DEBUG_PORT = 5567
- private val supportedSparkVersions = Set[String]("2.4.0", "2.4.1")
+ private val supportedSparkVersions = Set[String]("2.4.0", "2.4.1", "2.4.3")
val SPARK_VERSION = DotnetUtils.normalizeSparkVersion(spark.SPARK_VERSION)