From 03b79393e71910a33a39864e563fcbeb2de56658 Mon Sep 17 00:00:00 2001 From: Niharika Dutta Date: Sun, 19 Apr 2020 22:31:05 -0700 Subject: [PATCH 01/20] Adding section for UDF serialization --- docs/broadcast-guide.md | 92 +++++++++++++++++++++ docs/udf-guide.md | 172 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 264 insertions(+) create mode 100644 docs/broadcast-guide.md create mode 100644 docs/udf-guide.md diff --git a/docs/broadcast-guide.md b/docs/broadcast-guide.md new file mode 100644 index 000000000..4286c569e --- /dev/null +++ b/docs/broadcast-guide.md @@ -0,0 +1,92 @@ +# Guide to using Broadcast Variables + +This is a guide to show how to use broadcast variables in .NET for Apache Spark. + +## What are Broadcast Variables + +[Broadcast variables in Apache Spark](https://spark.apache.org/docs/2.2.0/rdd-programming-guide.html#broadcast-variables) are a mechanism for sharing variables across executors that are meant to be read-only. They allow the programmer to keep a read-only variable cached on each machine rather than shipping a copy of it with tasks. They can be used, for example, to give every node a copy of a large input dataset in an efficient manner. + +### How to use broadcast variables in .NET for Apache Spark + +Broadcast variables are created from a variable `v` by calling `SparkContext.Broadcast(v)`. The broadcast variable is a wrapper around `v`, and its value can be accessed by calling the `Value()` method on it. + +Example: + +```csharp +string v = "Variable to be broadcasted"; +Broadcast bv = SparkContext.Broadcast(v); + +// Using the broadcast variable in a UDF: +Func udf = Udf( + str => $"{str}: {bv.Value()}"); +``` + +The type of broadcast variable is captured by using Generics in C#, as can be seen in the above example. + +### Deleting broadcast variables + +The broadcast variable can be deleted from all executors by calling the `Destroy()` function on it. + +```csharp +// Destroying the broadcast variable bv: +bv.Destroy(); +``` + +> Note: `Destroy` deletes all data and metadata related to the broadcast variable. Use this with caution- once a broadcast variable has been destroyed, it cannot be used again. + +#### Caveat of using Destroy + +One important thing to keep in mind while using broadcast variables in UDFs is to limit the scope of the variable to only the UDF that is referencing it. The [guide to using UDFs](udf-guide.md) describes this phenomenon in detail. This is especially crucial when calling `Destroy` on the broadcast variable. If the broadcast variable that has been destroyed is visible to or accessible from other UDFs, it gets picked up for serialization by all those UDFs, even if it is not being referenced by them. This will throw an error as .NET for Apache Spark is not able to serialize the destroyed broadcast variable. + +Example to demonstrate: + +```csharp +string v = "Variable to be broadcasted"; +Broadcast bv = SparkContext.Broadcast(v); + +// Using the broadcast variable in a UDF: +Func udf1 = Udf( + str => $"{str}: {bv.Value()}"); + +// Destroying bv +bv.Destroy(); + +// Calling udf1 after destroying bv throws the following expected exception: +// org.apache.spark.SparkException: Attempted to use Broadcast(0) after it was destroyed +df.Select(udf1(df["_1"])).Show(); + +// Different UDF udf2 that is not referencing bv +Func udf2 = Udf( + str => $"{str}: not referencing broadcast variable"); + +// Calling udf2 throws the following (unexpected) exception: +// [Error] [JvmBridge] org.apache.spark.SparkException: Task not serializable +df.Select(udf2(df["_1"])).Show(); +``` + +The recommended way of implementing above desired behavior: + +```csharp +string v = "Variable to be broadcasted"; +// Restricting the visibility of bv to only the UDF referencing it +{ + Broadcast bv = SparkContext.Broadcast(v); + + // Using the broadcast variable in a UDF: + Func udf1 = Udf( + str => $"{str}: {bv.Value()}"); + + // Destroying bv + bv.Destroy(); +} + +// Different UDF udf2 that is not referencing bv +Func udf2 = Udf( + str => $"{str}: not referencing broadcast variable"); + +// Calling udf2 works fine as expected +df.Select(udf2(df["_1"])).Show(); +``` + This ensures that destroying `bv` doesn't affect calling `udf2` because of unexpected serialization behavior. + + Broadcast variables are very useful for transmitting read-only data to all executors, as the data is sent only once and this gives huge performance benefits when compared with using local variables that get shipped to the executors with each task. Please refer to the [official documentation](https://spark.apache.org/docs/2.2.0/rdd-programming-guide.html#broadcast-variables) to get a deeper understanding of broadcast variables and why they are used. \ No newline at end of file diff --git a/docs/udf-guide.md b/docs/udf-guide.md new file mode 100644 index 000000000..bb308815d --- /dev/null +++ b/docs/udf-guide.md @@ -0,0 +1,172 @@ +# Guide to User-Defined Functions (UDFs) + +This is a guide to show how to use UDFs in .NET for Apache Spark. + +## What are UDFs + +[User-Defined Functions (UDFs)](https://spark.apache.org/docs/latest/api/java/org/apache/spark/sql/expressions/UserDefinedFunction.html) are a feature of Spark that allow developers to use custom functions to extend the system's built-in functionality. They transform values from a single row within a table to produce a single corresponding output value per row based on the logic defined in the UDF. + +Let's take the following as an example for a UDF definition: + +```csharp +string s1 = "hello"; +Func udf = Udf( + str => $"{s1} {str}"); + +``` +The above defined UDF takes a `string` as an input (in the form of a [Column](https://github.com/dotnet/spark/blob/master/src/csharp/Microsoft.Spark/Sql/Column.cs#L14) of a [Dataframe](https://github.com/dotnet/spark/blob/master/src/csharp/Microsoft.Spark/Sql/DataFrame.cs#L24)), and returns a `string` with `hello` appended in front of the input. + +For a sample Dataframe, let's take the following Dataframe `df`: + +```text ++-------+ +| name| ++-------+ +|Michael| +| Andy| +| Justin| ++-------+ +``` + +Now let's apply the above defined `udf` to the dataframe `df`: + +```csharp +DataFrame udfResult = df.Select(udf(df["name"])); +``` + +This would return the below as the Dataframe `udfResult`: + +```text ++-------------+ +| name| ++-------------+ +|hello Michael| +| hello Andy| +| hello Justin| ++-------------+ +``` +To get a better understanding of how to implement UDFs, please take a look at the [UDF helper functions](https://github.com/dotnet/spark/blob/master/src/csharp/Microsoft.Spark/Sql/Functions.cs#L3616) and some [test examples](https://github.com/dotnet/spark/blob/master/src/csharp/Microsoft.Spark.E2ETest/UdfTests/UdfSimpleTypesTests.cs#L49). + +## UDF serialization + +Since UDFs are functions that need to be executed on the workers, they have to be serialized and sent to the workers as part of the payload from the driver. This involves serializing the [delegate](https://docs.microsoft.com/en-us/dotnet/csharp/programming-guide/delegates/) which is a reference to the method, along with its [target](https://docs.microsoft.com/en-us/dotnet/api/system.delegate.target?view=netframework-4.8) which is the class instance on which the current delegate invokes the instance method. Please take a look at this [code](https://github.com/dotnet/spark/blob/master/src/csharp/Microsoft.Spark/Utils/CommandSerDe.cs#L149) to get a better understanding of how UDF serialization is being done. + +## Good to know while implementing UDFs + +One behavior to be aware of while implementing UDFs in .NET for Apache Spark is how the target of the UDF gets serialized. .NET for Apache Spark uses .NET Core, which does not support serializing delegates, so it is instead done by using reflection to serialize the target where the delegate is defined. When multiple delegates are defined in a common scope, they have a shared closure that becomes the target of reflection for serialization. Let's take an example to illustrate what that means. + +The following code snippet defines two string variables that are being referenced in two function delegates, that just return the respective strings as result: + +```csharp +using System; + +public class C { + public void M() { + string s1 = "s1"; + string s2 = "s2"; + Func a = str => s1; + Func b = str => s2; + } +} +``` + +The above C# code generates the following C# disassembly (credit source: [sharplab.io](sharplab.io)) code from the compiler: + +```csharp +public class C +{ + [CompilerGenerated] + private sealed class <>c__DisplayClass0_0 + { + public string s1; + + public string s2; + + internal string b__0(string str) + { + return s1; + } + + internal string b__1(string str) + { + return s2; + } + } + + public void M() + { + <>c__DisplayClass0_0 <>c__DisplayClass0_ = new <>c__DisplayClass0_0(); + <>c__DisplayClass0_.s1 = "s1"; + <>c__DisplayClass0_.s2 = "s2"; + Func func = new Func(<>c__DisplayClass0_.b__0); + Func func2 = new Func(<>c__DisplayClass0_.b__1); + } +} +``` +As can be seen in the above IL code, both `func` and `func2` share the same closure `<>c__DisplayClass0_0`, which is the target that is serialized when serializing the delegates `func` and `func2`. Hence, even though `Func a` is only referencing `s1`, `s2` also gets serialized when sending over the bytes to the workers. + +This can lead to some unexpected behaviors at runtime (like in the case of using [broadcast variables](broadcast-guide.md)), which is why we recommend restricting the visibility of the variables used in a function to that function's scope. +Taking the above example to better explain what that means: + +Recommended user code to implement desired behavior of previous code snippet: + +```csharp +using System; + +public class C { + public void M() { + { + string s1 = "s1"; + Func a = str => s1; + } + { + string s2 = "s2"; + Func b = str => s2; + } + } +} +``` + +The above C# code generates the following C# disassembly (credit source: [sharplab.io](sharplab.io)) code from the compiler: + +```csharp +public class C +{ + [CompilerGenerated] + private sealed class <>c__DisplayClass0_0 + { + public string s1; + + internal string b__0(string str) + { + return s1; + } + } + + [CompilerGenerated] + private sealed class <>c__DisplayClass0_1 + { + public string s2; + + internal string b__1(string str) + { + return s2; + } + } + + public void M() + { + <>c__DisplayClass0_0 <>c__DisplayClass0_ = new <>c__DisplayClass0_0(); + <>c__DisplayClass0_.s1 = "s1"; + Func func = new Func(<>c__DisplayClass0_.b__0); + <>c__DisplayClass0_1 <>c__DisplayClass0_2 = new <>c__DisplayClass0_1(); + <>c__DisplayClass0_2.s2 = "s2"; + Func func2 = new Func(<>c__DisplayClass0_2.b__1); + } +} +``` + +Here we see that `func` and `func2` no longer share a closure and have their own separate closures `<>c__DisplayClass0_0` and `<>c__DisplayClass0_1` respectively. When used as the target for serialization, nothing other than the referenced variables will get serialized for the delegate. + +This above behavior is important to keep in mind while implementing multiple UDFs in a common scope. +To learn more about UDFs in general, please review the following articles that explain UDFs and how to use them: [UDFs in databricks(scala)](https://docs.databricks.com/spark/latest/spark-sql/udf-scala.html), [Spark UDFs and some gotchas](https://medium.com/@achilleus/spark-udfs-we-can-use-them-but-should-we-use-them-2c5a561fde6d). \ No newline at end of file From 4ef693dbf7616b738a6ae70d1e9dc8c12dd8e5d3 Mon Sep 17 00:00:00 2001 From: Niharika Dutta Date: Sun, 19 Apr 2020 22:32:56 -0700 Subject: [PATCH 02/20] removing guides from master --- docs/broadcast-guide.md | 92 --------------------- docs/udf-guide.md | 172 ---------------------------------------- 2 files changed, 264 deletions(-) delete mode 100644 docs/broadcast-guide.md delete mode 100644 docs/udf-guide.md diff --git a/docs/broadcast-guide.md b/docs/broadcast-guide.md deleted file mode 100644 index 4286c569e..000000000 --- a/docs/broadcast-guide.md +++ /dev/null @@ -1,92 +0,0 @@ -# Guide to using Broadcast Variables - -This is a guide to show how to use broadcast variables in .NET for Apache Spark. - -## What are Broadcast Variables - -[Broadcast variables in Apache Spark](https://spark.apache.org/docs/2.2.0/rdd-programming-guide.html#broadcast-variables) are a mechanism for sharing variables across executors that are meant to be read-only. They allow the programmer to keep a read-only variable cached on each machine rather than shipping a copy of it with tasks. They can be used, for example, to give every node a copy of a large input dataset in an efficient manner. - -### How to use broadcast variables in .NET for Apache Spark - -Broadcast variables are created from a variable `v` by calling `SparkContext.Broadcast(v)`. The broadcast variable is a wrapper around `v`, and its value can be accessed by calling the `Value()` method on it. - -Example: - -```csharp -string v = "Variable to be broadcasted"; -Broadcast bv = SparkContext.Broadcast(v); - -// Using the broadcast variable in a UDF: -Func udf = Udf( - str => $"{str}: {bv.Value()}"); -``` - -The type of broadcast variable is captured by using Generics in C#, as can be seen in the above example. - -### Deleting broadcast variables - -The broadcast variable can be deleted from all executors by calling the `Destroy()` function on it. - -```csharp -// Destroying the broadcast variable bv: -bv.Destroy(); -``` - -> Note: `Destroy` deletes all data and metadata related to the broadcast variable. Use this with caution- once a broadcast variable has been destroyed, it cannot be used again. - -#### Caveat of using Destroy - -One important thing to keep in mind while using broadcast variables in UDFs is to limit the scope of the variable to only the UDF that is referencing it. The [guide to using UDFs](udf-guide.md) describes this phenomenon in detail. This is especially crucial when calling `Destroy` on the broadcast variable. If the broadcast variable that has been destroyed is visible to or accessible from other UDFs, it gets picked up for serialization by all those UDFs, even if it is not being referenced by them. This will throw an error as .NET for Apache Spark is not able to serialize the destroyed broadcast variable. - -Example to demonstrate: - -```csharp -string v = "Variable to be broadcasted"; -Broadcast bv = SparkContext.Broadcast(v); - -// Using the broadcast variable in a UDF: -Func udf1 = Udf( - str => $"{str}: {bv.Value()}"); - -// Destroying bv -bv.Destroy(); - -// Calling udf1 after destroying bv throws the following expected exception: -// org.apache.spark.SparkException: Attempted to use Broadcast(0) after it was destroyed -df.Select(udf1(df["_1"])).Show(); - -// Different UDF udf2 that is not referencing bv -Func udf2 = Udf( - str => $"{str}: not referencing broadcast variable"); - -// Calling udf2 throws the following (unexpected) exception: -// [Error] [JvmBridge] org.apache.spark.SparkException: Task not serializable -df.Select(udf2(df["_1"])).Show(); -``` - -The recommended way of implementing above desired behavior: - -```csharp -string v = "Variable to be broadcasted"; -// Restricting the visibility of bv to only the UDF referencing it -{ - Broadcast bv = SparkContext.Broadcast(v); - - // Using the broadcast variable in a UDF: - Func udf1 = Udf( - str => $"{str}: {bv.Value()}"); - - // Destroying bv - bv.Destroy(); -} - -// Different UDF udf2 that is not referencing bv -Func udf2 = Udf( - str => $"{str}: not referencing broadcast variable"); - -// Calling udf2 works fine as expected -df.Select(udf2(df["_1"])).Show(); -``` - This ensures that destroying `bv` doesn't affect calling `udf2` because of unexpected serialization behavior. - - Broadcast variables are very useful for transmitting read-only data to all executors, as the data is sent only once and this gives huge performance benefits when compared with using local variables that get shipped to the executors with each task. Please refer to the [official documentation](https://spark.apache.org/docs/2.2.0/rdd-programming-guide.html#broadcast-variables) to get a deeper understanding of broadcast variables and why they are used. \ No newline at end of file diff --git a/docs/udf-guide.md b/docs/udf-guide.md deleted file mode 100644 index bb308815d..000000000 --- a/docs/udf-guide.md +++ /dev/null @@ -1,172 +0,0 @@ -# Guide to User-Defined Functions (UDFs) - -This is a guide to show how to use UDFs in .NET for Apache Spark. - -## What are UDFs - -[User-Defined Functions (UDFs)](https://spark.apache.org/docs/latest/api/java/org/apache/spark/sql/expressions/UserDefinedFunction.html) are a feature of Spark that allow developers to use custom functions to extend the system's built-in functionality. They transform values from a single row within a table to produce a single corresponding output value per row based on the logic defined in the UDF. - -Let's take the following as an example for a UDF definition: - -```csharp -string s1 = "hello"; -Func udf = Udf( - str => $"{s1} {str}"); - -``` -The above defined UDF takes a `string` as an input (in the form of a [Column](https://github.com/dotnet/spark/blob/master/src/csharp/Microsoft.Spark/Sql/Column.cs#L14) of a [Dataframe](https://github.com/dotnet/spark/blob/master/src/csharp/Microsoft.Spark/Sql/DataFrame.cs#L24)), and returns a `string` with `hello` appended in front of the input. - -For a sample Dataframe, let's take the following Dataframe `df`: - -```text -+-------+ -| name| -+-------+ -|Michael| -| Andy| -| Justin| -+-------+ -``` - -Now let's apply the above defined `udf` to the dataframe `df`: - -```csharp -DataFrame udfResult = df.Select(udf(df["name"])); -``` - -This would return the below as the Dataframe `udfResult`: - -```text -+-------------+ -| name| -+-------------+ -|hello Michael| -| hello Andy| -| hello Justin| -+-------------+ -``` -To get a better understanding of how to implement UDFs, please take a look at the [UDF helper functions](https://github.com/dotnet/spark/blob/master/src/csharp/Microsoft.Spark/Sql/Functions.cs#L3616) and some [test examples](https://github.com/dotnet/spark/blob/master/src/csharp/Microsoft.Spark.E2ETest/UdfTests/UdfSimpleTypesTests.cs#L49). - -## UDF serialization - -Since UDFs are functions that need to be executed on the workers, they have to be serialized and sent to the workers as part of the payload from the driver. This involves serializing the [delegate](https://docs.microsoft.com/en-us/dotnet/csharp/programming-guide/delegates/) which is a reference to the method, along with its [target](https://docs.microsoft.com/en-us/dotnet/api/system.delegate.target?view=netframework-4.8) which is the class instance on which the current delegate invokes the instance method. Please take a look at this [code](https://github.com/dotnet/spark/blob/master/src/csharp/Microsoft.Spark/Utils/CommandSerDe.cs#L149) to get a better understanding of how UDF serialization is being done. - -## Good to know while implementing UDFs - -One behavior to be aware of while implementing UDFs in .NET for Apache Spark is how the target of the UDF gets serialized. .NET for Apache Spark uses .NET Core, which does not support serializing delegates, so it is instead done by using reflection to serialize the target where the delegate is defined. When multiple delegates are defined in a common scope, they have a shared closure that becomes the target of reflection for serialization. Let's take an example to illustrate what that means. - -The following code snippet defines two string variables that are being referenced in two function delegates, that just return the respective strings as result: - -```csharp -using System; - -public class C { - public void M() { - string s1 = "s1"; - string s2 = "s2"; - Func a = str => s1; - Func b = str => s2; - } -} -``` - -The above C# code generates the following C# disassembly (credit source: [sharplab.io](sharplab.io)) code from the compiler: - -```csharp -public class C -{ - [CompilerGenerated] - private sealed class <>c__DisplayClass0_0 - { - public string s1; - - public string s2; - - internal string b__0(string str) - { - return s1; - } - - internal string b__1(string str) - { - return s2; - } - } - - public void M() - { - <>c__DisplayClass0_0 <>c__DisplayClass0_ = new <>c__DisplayClass0_0(); - <>c__DisplayClass0_.s1 = "s1"; - <>c__DisplayClass0_.s2 = "s2"; - Func func = new Func(<>c__DisplayClass0_.b__0); - Func func2 = new Func(<>c__DisplayClass0_.b__1); - } -} -``` -As can be seen in the above IL code, both `func` and `func2` share the same closure `<>c__DisplayClass0_0`, which is the target that is serialized when serializing the delegates `func` and `func2`. Hence, even though `Func a` is only referencing `s1`, `s2` also gets serialized when sending over the bytes to the workers. - -This can lead to some unexpected behaviors at runtime (like in the case of using [broadcast variables](broadcast-guide.md)), which is why we recommend restricting the visibility of the variables used in a function to that function's scope. -Taking the above example to better explain what that means: - -Recommended user code to implement desired behavior of previous code snippet: - -```csharp -using System; - -public class C { - public void M() { - { - string s1 = "s1"; - Func a = str => s1; - } - { - string s2 = "s2"; - Func b = str => s2; - } - } -} -``` - -The above C# code generates the following C# disassembly (credit source: [sharplab.io](sharplab.io)) code from the compiler: - -```csharp -public class C -{ - [CompilerGenerated] - private sealed class <>c__DisplayClass0_0 - { - public string s1; - - internal string b__0(string str) - { - return s1; - } - } - - [CompilerGenerated] - private sealed class <>c__DisplayClass0_1 - { - public string s2; - - internal string b__1(string str) - { - return s2; - } - } - - public void M() - { - <>c__DisplayClass0_0 <>c__DisplayClass0_ = new <>c__DisplayClass0_0(); - <>c__DisplayClass0_.s1 = "s1"; - Func func = new Func(<>c__DisplayClass0_.b__0); - <>c__DisplayClass0_1 <>c__DisplayClass0_2 = new <>c__DisplayClass0_1(); - <>c__DisplayClass0_2.s2 = "s2"; - Func func2 = new Func(<>c__DisplayClass0_2.b__1); - } -} -``` - -Here we see that `func` and `func2` no longer share a closure and have their own separate closures `<>c__DisplayClass0_0` and `<>c__DisplayClass0_1` respectively. When used as the target for serialization, nothing other than the referenced variables will get serialized for the delegate. - -This above behavior is important to keep in mind while implementing multiple UDFs in a common scope. -To learn more about UDFs in general, please review the following articles that explain UDFs and how to use them: [UDFs in databricks(scala)](https://docs.databricks.com/spark/latest/spark-sql/udf-scala.html), [Spark UDFs and some gotchas](https://medium.com/@achilleus/spark-udfs-we-can-use-them-but-should-we-use-them-2c5a561fde6d). \ No newline at end of file From 6bab99604db5cc8b8528b54216085afb96cbaff7 Mon Sep 17 00:00:00 2001 From: GOEddieUK Date: Mon, 27 Jul 2020 21:10:51 +0100 Subject: [PATCH 03/20] CountVectorizer --- .../ML/Feature/CountVectorizerModelTests.cs | 73 +++++++ .../ML/Feature/CountVectorizerTests.cs | 70 +++++++ .../ML/Feature/CountVectorizer.cs | 195 ++++++++++++++++++ .../ML/Feature/CountVectorizerModel.cs | 170 +++++++++++++++ 4 files changed, 508 insertions(+) create mode 100644 src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerModelTests.cs create mode 100644 src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerTests.cs create mode 100644 src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs create mode 100644 src/csharp/Microsoft.Spark/ML/Feature/CountVectorizerModel.cs diff --git a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerModelTests.cs b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerModelTests.cs new file mode 100644 index 000000000..3c3132dd9 --- /dev/null +++ b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerModelTests.cs @@ -0,0 +1,73 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.Spark.ML.Feature; +using Microsoft.Spark.Sql; +using Microsoft.Spark.UnitTest.TestUtils; +using Xunit; + +namespace Microsoft.Spark.E2ETest.IpcTests.ML.Feature +{ + [Collection("Spark E2E Tests")] + public class CountVectorizerModelTests + { + private readonly SparkSession _spark; + + public CountVectorizerModelTests(SparkFixture fixture) + { + _spark = fixture.Spark; + } + + [Fact] + public void Test_CountVectorizerModel() + { + DataFrame input = _spark.Sql("SELECT array('hello', 'I', 'AM', 'a', 'string', 'TO', " + + "'TOKENIZE') as input from range(100)"); + + const string inputColumn = "input"; + const string outputColumn = "output"; + const double minTf = 10.0; + const bool binary = false; + + List vocabulary = new List() + { + "hello", + "I", + "AM", + "TO", + "TOKENIZE" + }; + + var countVectorizerModel = new CountVectorizerModel(vocabulary); + + Assert.IsType(new CountVectorizerModel("my-uid", vocabulary)); + + countVectorizerModel = countVectorizerModel + .SetInputCol(inputColumn) + .SetOutputCol(outputColumn) + .SetMinTF(minTf) + .SetBinary(binary); + + Assert.Equal(inputColumn, countVectorizerModel.GetInputCol()); + Assert.Equal(outputColumn, countVectorizerModel.GetOutputCol()); + Assert.Equal(minTf, countVectorizerModel.GetMinTF()); + Assert.Equal(binary, countVectorizerModel.GetBinary()); + using (var tempDirectory = new TemporaryDirectory()) + { + string savePath = Path.Join(tempDirectory.Path, "countVectorizerModel"); + countVectorizerModel.Save(savePath); + + CountVectorizerModel loadedModel = CountVectorizerModel.Load(savePath); + Assert.Equal(countVectorizerModel.Uid(), loadedModel.Uid()); + } + + Assert.IsType(countVectorizerModel.GetVocabSize()); + Assert.NotEmpty(countVectorizerModel.ExplainParams()); + Assert.NotEmpty(countVectorizerModel.ToString()); + } + } +} diff --git a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerTests.cs b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerTests.cs new file mode 100644 index 000000000..d54bfe376 --- /dev/null +++ b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerTests.cs @@ -0,0 +1,70 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using Microsoft.Spark.ML.Feature; +using Microsoft.Spark.Sql; +using Microsoft.Spark.UnitTest.TestUtils; +using Xunit; + +namespace Microsoft.Spark.E2ETest.IpcTests.ML.Feature +{ + [Collection("Spark E2E Tests")] + public class CountVectorizerTests + { + private readonly SparkSession _spark; + + public CountVectorizerTests(SparkFixture fixture) + { + _spark = fixture.Spark; + } + + [Fact] + public void Test_CountVectorizer() + { + DataFrame input = _spark.Sql("SELECT array('hello', 'I', 'AM', 'a', 'string', 'TO', " + + "'TOKENIZE') as input from range(100)"); + + const string inputColumn = "input"; + const string outputColumn = "output"; + const double minDf = 1; + const double maxDf = 100; + const double minTf = 10; + const int vocabSize = 10000; + const bool binary = false; + + var countVectorizer = new CountVectorizer(); + + countVectorizer + .SetInputCol(inputColumn) + .SetOutputCol(outputColumn) + .SetMinDF(minDf) + .SetMaxDF(maxDf) + .SetMinTF(minTf) + .SetVocabSize(vocabSize); + + Assert.IsType(countVectorizer.Fit(input)); + Assert.Equal(inputColumn, countVectorizer.GetInputCol()); + Assert.Equal(outputColumn, countVectorizer.GetOutputCol()); + Assert.Equal(minDf, countVectorizer.GetMinDF()); + Assert.Equal(maxDf, countVectorizer.GetMaxDF()); + Assert.Equal(minTf, countVectorizer.GetMinTF()); + Assert.Equal(vocabSize, countVectorizer.GetVocabSize()); + Assert.Equal(binary, countVectorizer.GetBinary()); + + using (var tempDirectory = new TemporaryDirectory()) + { + string savePath = Path.Join(tempDirectory.Path, "countVectorizer"); + countVectorizer.Save(savePath); + + CountVectorizer loadedVectorizer = CountVectorizer.Load(savePath); + Assert.Equal(countVectorizer.Uid(), loadedVectorizer.Uid()); + } + + Assert.NotEmpty(countVectorizer.ExplainParams()); + Assert.NotEmpty(countVectorizer.ToString()); + } + } +} diff --git a/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs b/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs new file mode 100644 index 000000000..41e0dbdd0 --- /dev/null +++ b/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs @@ -0,0 +1,195 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Spark.Interop; +using Microsoft.Spark.Interop.Ipc; +using Microsoft.Spark.Sql; + +namespace Microsoft.Spark.ML.Feature +{ + public class CountVectorizer : FeatureBase, IJvmObjectReferenceProvider + { + private static readonly string s_countVectorizerClassName = + "org.apache.spark.ml.feature.CountVectorizer"; + + /// + /// Create a without any parameters + /// + public CountVectorizer() : base(s_countVectorizerClassName) + { + } + + /// + /// Create a with a UID that is used to give the + /// a unique ID + /// + /// An immutable unique ID for the object and its derivatives. + public CountVectorizer(string uid) : base(s_countVectorizerClassName, uid) + { + } + + internal CountVectorizer(JvmObjectReference jvmObject) : base(jvmObject) + { + } + + JvmObjectReference IJvmObjectReferenceProvider.Reference => _jvmObject; + + /// Fits a model to the input data. + /// The to fit the model to. + /// + public CountVectorizerModel Fit(DataFrame dataFrame) => + new CountVectorizerModel((JvmObjectReference)_jvmObject.Invoke("fit", dataFrame)); + + /// + /// Loads the that was previously saved using Save + /// + /// + /// The path the previous was saved to + /// + /// New object + public static CountVectorizer Load(string path) => + WrapAsType((JvmObjectReference) + SparkEnvironment.JvmBridge.CallStaticJavaMethod( + s_countVectorizerClassName,"load", path)); + + /// + /// Gets the binary toggle to control the output vector values. If True, all nonzero counts + /// (after minTF filter applied) are set to 1. This is useful for discrete probabilistic + /// models that model binary events rather than integer counts. Default: false + /// + /// boolean + public bool GetBinary() => (bool)_jvmObject.Invoke("getBinary"); + + /// + /// Sets the binary toggle to control the output vector values. If True, all nonzero counts + /// (after minTF filter applied) are set to 1. This is useful for discrete probabilistic + /// models that model binary events rather than integer counts. Default: false + /// + /// Turn the binary toggle on or off + /// with the new binary toggle value set + public CountVectorizer SetBinary(bool value) => + WrapAsCountVectorizer((JvmObjectReference)_jvmObject.Invoke("setBinary", value)); + + private static CountVectorizer WrapAsCountVectorizer(object obj) => + new CountVectorizer((JvmObjectReference)obj); + + /// + /// Gets the column that the should read from and convert + /// into buckets. This would have been set by SetInputCol + /// + /// string, the input column + public string GetInputCol() => _jvmObject.Invoke("getInputCol") as string; + + /// + /// Sets the column that the should read from. + /// + /// The name of the column to as the source. + /// with the input column set + public CountVectorizer SetInputCol(string value) => + WrapAsCountVectorizer((JvmObjectReference)_jvmObject.Invoke("setInputCol", value)); + + /// + /// The will create a new column in the DataFrame, this is + /// the name of the new column. + /// + /// The name of the output column. + public string GetOutputCol() => _jvmObject.Invoke("getOutputCol") as string; + + /// + /// The will create a new column in the DataFrame, this + /// is the name of the new column. + /// + /// The name of the output column which will be created. + /// New with the output column set + public CountVectorizer SetOutputCol(string value) => + WrapAsCountVectorizer((JvmObjectReference)_jvmObject.Invoke("setOutputCol", value)); + + /// + /// Gets the maximum number of different documents a term could appear in to be included in + /// the vocabulary. A term that appears more than the threshold will be ignored. If this is + /// an integer greater than or equal to 1, this specifies the maximum number of documents + /// the term could appear in; if this is a double in [0,1), then this specifies the maximum + /// fraction of documents the term could appear in. + /// + /// The maximum document term frequency + public double GetMaxDF() => (double)_jvmObject.Invoke("getMaxDF"); + + /// + /// Sets the maximum number of different documents a term could appear in to be included in + /// the vocabulary. A term that appears more than the threshold will be ignored. If this is + /// an integer greater than or equal to 1, this specifies the maximum number of documents + /// the term could appear in; if this is a double in [0,1), then this specifies the maximum + /// fraction of documents the term could appear in. + /// + /// The maximum document term frequency + /// New with the max df value set + public CountVectorizer SetMaxDF(double value) => + WrapAsCountVectorizer((JvmObjectReference)_jvmObject.Invoke("setMaxDF", value)); + + /// + /// Gets the minimum number of different documents a term must appear in to be included in + /// the vocabulary. If this is an integer greater than or equal to 1, this specifies the + /// number of documents the term must appear in; if this is a double in [0,1), then this + /// specifies the fraction of documents. + /// + /// The minimum document term frequency + public double GetMinDF() => (double)_jvmObject.Invoke("getMinDF"); + + /// + /// Sets the minimum number of different documents a term must appear in to be included in + /// the vocabulary. If this is an integer greater than or equal to 1, this specifies the + /// number of documents the term must appear in; if this is a double in [0,1), then this + /// specifies the fraction of documents. + /// + /// The minimum document term frequency + /// New with the min df value set + public CountVectorizer SetMinDF(double value) => + WrapAsCountVectorizer((JvmObjectReference)_jvmObject.Invoke("setMinDF", value)); + + /// + /// Filter to ignore rare words in a document. For each document, terms with + /// frequency/count less than the given threshold are ignored. If this is an integer + /// greater than or equal to 1, then this specifies a count (of times the term must appear + /// in the document); if this is a double in [0,1), then this specifies a fraction (out of + /// the document's token count). + /// + /// Note that the parameter is only used in transform of CountVectorizerModel and does not + /// affect fitting. + /// + /// Minimum term frequency + public double GetMinTF() => (double)_jvmObject.Invoke("getMinTF"); + + /// + /// Filter to ignore rare words in a document. For each document, terms with + /// frequency/count less than the given threshold are ignored. If this is an integer + /// greater than or equal to 1, then this specifies a count (of times the term must appear + /// in the document); if this is a double in [0,1), then this specifies a fraction (out of + /// the document's token count). + /// + /// Note that the parameter is only used in transform of CountVectorizerModel and does not + /// affect fitting. + /// + /// Minimum term frequency + /// New with the min term frequency set + public CountVectorizer SetMinTF(double value) => + WrapAsCountVectorizer((JvmObjectReference)_jvmObject.Invoke("setMinTF", value)); + + /// + /// Gets the max size of the vocabulary. CountVectorizer will build a vocabulary that only + /// considers the top vocabSize terms ordered by term frequency across the corpus. + /// + /// The max size of the vocabulary + public int GetVocabSize() => (int)_jvmObject.Invoke("getVocabSize"); + + /// + /// Sets the max size of the vocabulary. will build a + /// vocabulary that only considers the top vocabSize terms ordered by term frequency across + /// the corpus. + /// + /// The max vocabulary size + /// with the max vocab value set + public CountVectorizer SetVocabSize(int value) => + WrapAsCountVectorizer(_jvmObject.Invoke("setVocabSize", value)); + } +} diff --git a/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizerModel.cs b/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizerModel.cs new file mode 100644 index 000000000..8a6e427df --- /dev/null +++ b/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizerModel.cs @@ -0,0 +1,170 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using Microsoft.Spark.Interop; +using Microsoft.Spark.Interop.Ipc; + +namespace Microsoft.Spark.ML.Feature +{ + public class CountVectorizerModel : FeatureBase + , IJvmObjectReferenceProvider + { + private static readonly string s_countVectorizerModelClassName = + "org.apache.spark.ml.feature.CountVectorizerModel"; + + /// + /// Create a without any parameters + /// + /// The vocabulary to use + public CountVectorizerModel(List vocabulary) : + this(SparkEnvironment.JvmBridge.CallConstructor( + s_countVectorizerModelClassName, vocabulary)) + { + } + + /// + /// Create a with a UID that is used to give the + /// a unique ID + /// + /// An immutable unique ID for the object and its derivatives. + /// The vocabulary to use + public CountVectorizerModel(string uid, List vocabulary) : + this(SparkEnvironment.JvmBridge.CallConstructor( + s_countVectorizerModelClassName, uid, vocabulary)) + { + } + + internal CountVectorizerModel(JvmObjectReference jvmObject) : base(jvmObject) + { + } + + JvmObjectReference IJvmObjectReferenceProvider.Reference => _jvmObject; + + /// + /// Loads the that was previously saved using Save + /// + /// + /// The path the previous was saved to + /// + /// New object + public static CountVectorizerModel Load(string path) => + WrapAsType((JvmObjectReference) + SparkEnvironment.JvmBridge.CallStaticJavaMethod( + s_countVectorizerModelClassName,"load", path)); + + /// + /// Gets the binary toggle to control the output vector values. If True, all nonzero counts + /// (after minTF filter applied) are set to 1. This is useful for discrete probabilistic + /// models that model binary events rather than integer counts. Default: false + /// + /// boolean + public bool GetBinary() => (bool)_jvmObject.Invoke("getBinary"); + + /// + /// Sets the binary toggle to control the output vector values. If True, all nonzero counts + /// (after minTF filter applied) are set to 1. This is useful for discrete probabilistic + /// models that model binary events rather than integer counts. Default: false + /// + /// Turn the binary toggle on or off + /// + /// with the new binary toggle value set + /// + public CountVectorizerModel SetBinary(bool value) => + WrapAsCountVectorizerModel((JvmObjectReference)_jvmObject.Invoke("setBinary", value)); + + private static CountVectorizerModel WrapAsCountVectorizerModel(object obj) => + new CountVectorizerModel((JvmObjectReference)obj); + + /// + /// Gets the column that the should read from and + /// convert into buckets. This would have been set by SetInputCol + /// + /// string, the input column + public string GetInputCol() => _jvmObject.Invoke("getInputCol") as string; + + /// + /// Sets the column that the should read from. + /// + /// The name of the column to as the source. + /// with the input column set + public CountVectorizerModel SetInputCol(string value) => + WrapAsCountVectorizerModel( + (JvmObjectReference)_jvmObject.Invoke("setInputCol", value)); + + /// + /// The will create a new column in the DataFrame, this + /// is the name of the new column. + /// + /// The name of the output column. + public string GetOutputCol() => _jvmObject.Invoke("getOutputCol") as string; + + /// + /// The will create a new column in the DataFrame, + /// this is the name of the new column. + /// + /// The name of the output column which will be created. + /// New with the output column set + public CountVectorizerModel SetOutputCol(string value) => + WrapAsCountVectorizerModel( + (JvmObjectReference)_jvmObject.Invoke("setOutputCol", value)); + + /// + /// Gets the maximum number of different documents a term could appear in to be included in + /// the vocabulary. A term that appears more than the threshold will be ignored. If this is + /// an integer greater than or equal to 1, this specifies the maximum number of documents + /// the term could appear in; if this is a double in [0,1), then this specifies the maximum + /// fraction of documents the term could appear in. + /// + /// The maximum document term frequency + public double GetMaxDF() => (double)_jvmObject.Invoke("getMaxDF"); + + /// + /// Gets the minimum number of different documents a term must appear in to be included in + /// the vocabulary. If this is an integer greater than or equal to 1, this specifies the + /// number of documents the term must appear in; if this is a double in [0,1), then this + /// specifies the fraction of documents. + /// + /// The minimum document term frequency + public double GetMinDF() => (double)_jvmObject.Invoke("getMinDF"); + + /// + /// Filter to ignore rare words in a document. For each document, terms with + /// frequency/count less than the given threshold are ignored. If this is an integer + /// greater than or equal to 1, then this specifies a count (of times the term must appear + /// in the document); if this is a double in [0,1), then this specifies a fraction (out of + /// the document's token count). + /// + /// Note that the parameter is only used in transform of CountVectorizerModel and does not + /// affect fitting. + /// + /// Minimum term frequency + public double GetMinTF() => (double)_jvmObject.Invoke("getMinTF"); + + /// + /// Filter to ignore rare words in a document. For each document, terms with + /// frequency/count less than the given threshold are ignored. If this is an integer + /// greater than or equal to 1, then this specifies a count (of times the term must appear + /// in the document); if this is a double in [0,1), then this specifies a fraction (out of + /// the document's token count). + /// + /// Note that the parameter is only used in transform of CountVectorizerModel and does not + /// affect fitting. + /// + /// Minimum term frequency + /// + /// New with the min term frequency set + /// + public CountVectorizerModel SetMinTF(double value) => + WrapAsCountVectorizerModel((JvmObjectReference)_jvmObject.Invoke("setMinTF", value)); + + /// + /// Gets the max size of the vocabulary. will build a + /// vocabulary that only considers the top vocabSize terms ordered by term frequency across + /// the corpus. + /// + /// The max size of the vocabulary + public int GetVocabSize() => (int)_jvmObject.Invoke("getVocabSize"); + } +} From e2a566b1f4b29775be9b57616a258802e294f304 Mon Sep 17 00:00:00 2001 From: GOEddieUK Date: Mon, 27 Jul 2020 21:24:35 +0100 Subject: [PATCH 04/20] moving private methods to bottom --- src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs | 6 +++--- .../Microsoft.Spark/ML/Feature/CountVectorizerModel.cs | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs b/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs index 41e0dbdd0..cf68f7c4a 100644 --- a/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs +++ b/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs @@ -71,9 +71,6 @@ public static CountVectorizer Load(string path) => public CountVectorizer SetBinary(bool value) => WrapAsCountVectorizer((JvmObjectReference)_jvmObject.Invoke("setBinary", value)); - private static CountVectorizer WrapAsCountVectorizer(object obj) => - new CountVectorizer((JvmObjectReference)obj); - /// /// Gets the column that the should read from and convert /// into buckets. This would have been set by SetInputCol @@ -191,5 +188,8 @@ public CountVectorizer SetMinTF(double value) => /// with the max vocab value set public CountVectorizer SetVocabSize(int value) => WrapAsCountVectorizer(_jvmObject.Invoke("setVocabSize", value)); + + private static CountVectorizer WrapAsCountVectorizer(object obj) => + new CountVectorizer((JvmObjectReference)obj); } } diff --git a/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizerModel.cs b/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizerModel.cs index 8a6e427df..8e225a179 100644 --- a/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizerModel.cs +++ b/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizerModel.cs @@ -74,9 +74,6 @@ public static CountVectorizerModel Load(string path) => public CountVectorizerModel SetBinary(bool value) => WrapAsCountVectorizerModel((JvmObjectReference)_jvmObject.Invoke("setBinary", value)); - private static CountVectorizerModel WrapAsCountVectorizerModel(object obj) => - new CountVectorizerModel((JvmObjectReference)obj); - /// /// Gets the column that the should read from and /// convert into buckets. This would have been set by SetInputCol @@ -166,5 +163,8 @@ public CountVectorizerModel SetMinTF(double value) => /// /// The max size of the vocabulary public int GetVocabSize() => (int)_jvmObject.Invoke("getVocabSize"); + + private static CountVectorizerModel WrapAsCountVectorizerModel(object obj) => + new CountVectorizerModel((JvmObjectReference)obj); } } From 5f682a601ec783f1609e6fd6e32c4d83ff1491d1 Mon Sep 17 00:00:00 2001 From: GOEddieUK Date: Tue, 28 Jul 2020 20:47:31 +0100 Subject: [PATCH 05/20] changing wrap method --- src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs | 2 +- src/csharp/Microsoft.Spark/ML/Feature/CountVectorizerModel.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs b/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs index cf68f7c4a..b3fa0ef8a 100644 --- a/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs +++ b/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs @@ -49,7 +49,7 @@ public CountVectorizerModel Fit(DataFrame dataFrame) => /// /// New object public static CountVectorizer Load(string path) => - WrapAsType((JvmObjectReference) + WrapAsCountVectorizer((JvmObjectReference) SparkEnvironment.JvmBridge.CallStaticJavaMethod( s_countVectorizerClassName,"load", path)); diff --git a/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizerModel.cs b/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizerModel.cs index 8e225a179..52bbd72c3 100644 --- a/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizerModel.cs +++ b/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizerModel.cs @@ -50,7 +50,7 @@ internal CountVectorizerModel(JvmObjectReference jvmObject) : base(jvmObject) /// /// New object public static CountVectorizerModel Load(string path) => - WrapAsType((JvmObjectReference) + WrapAsCountVectorizerModel((JvmObjectReference) SparkEnvironment.JvmBridge.CallStaticJavaMethod( s_countVectorizerModelClassName,"load", path)); From 31371db73b4faa653c07fdb8082e7aed02c0a031 Mon Sep 17 00:00:00 2001 From: GOEddieUK Date: Fri, 31 Jul 2020 18:45:46 +0100 Subject: [PATCH 06/20] setting min version required --- .../IpcTests/ML/Feature/CountVectorizerTests.cs | 14 ++++++++++---- .../Microsoft.Spark/ML/Feature/CountVectorizer.cs | 2 ++ .../Microsoft.Spark/ML/Feature/FeatureBase.cs | 3 ++- src/csharp/Microsoft.Spark/Microsoft.Spark.csproj | 5 +---- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerTests.cs b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerTests.cs index d54bfe376..95b9bc504 100644 --- a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerTests.cs +++ b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerTests.cs @@ -4,6 +4,7 @@ using System; using System.IO; +using Microsoft.Spark.E2ETest.Utils; using Microsoft.Spark.ML.Feature; using Microsoft.Spark.Sql; using Microsoft.Spark.UnitTest.TestUtils; @@ -30,7 +31,6 @@ public void Test_CountVectorizer() const string inputColumn = "input"; const string outputColumn = "output"; const double minDf = 1; - const double maxDf = 100; const double minTf = 10; const int vocabSize = 10000; const bool binary = false; @@ -41,7 +41,6 @@ public void Test_CountVectorizer() .SetInputCol(inputColumn) .SetOutputCol(outputColumn) .SetMinDF(minDf) - .SetMaxDF(maxDf) .SetMinTF(minTf) .SetVocabSize(vocabSize); @@ -49,7 +48,6 @@ public void Test_CountVectorizer() Assert.Equal(inputColumn, countVectorizer.GetInputCol()); Assert.Equal(outputColumn, countVectorizer.GetOutputCol()); Assert.Equal(minDf, countVectorizer.GetMinDF()); - Assert.Equal(maxDf, countVectorizer.GetMaxDF()); Assert.Equal(minTf, countVectorizer.GetMinTF()); Assert.Equal(vocabSize, countVectorizer.GetVocabSize()); Assert.Equal(binary, countVectorizer.GetBinary()); @@ -65,6 +63,14 @@ public void Test_CountVectorizer() Assert.NotEmpty(countVectorizer.ExplainParams()); Assert.NotEmpty(countVectorizer.ToString()); - } + } + + [SkipIfSparkVersionIsLessThan(Versions.V2_4_0)] + public void CountVectorizer_MaxDF() + { + const double maxDf = 100; + CountVectorizer countVectorizer = new CountVectorizer().SetMaxDF(maxDf); + Assert.Equal(maxDf, countVectorizer.GetMaxDF()); + } } } diff --git a/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs b/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs index b3fa0ef8a..5689e19fd 100644 --- a/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs +++ b/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs @@ -110,6 +110,7 @@ public CountVectorizer SetOutputCol(string value) => /// fraction of documents the term could appear in. /// /// The maximum document term frequency + [Since(Versions.V2_4_0)] public double GetMaxDF() => (double)_jvmObject.Invoke("getMaxDF"); /// @@ -121,6 +122,7 @@ public CountVectorizer SetOutputCol(string value) => /// /// The maximum document term frequency /// New with the max df value set + [Since(Versions.V2_4_0)] public CountVectorizer SetMaxDF(double value) => WrapAsCountVectorizer((JvmObjectReference)_jvmObject.Invoke("setMaxDF", value)); diff --git a/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs b/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs index fcc90b43d..0895dace1 100644 --- a/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs +++ b/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs @@ -98,7 +98,7 @@ public Param.Param GetParam(string paramName) => public T Set(Param.Param param, object value) => WrapAsType((JvmObjectReference)_jvmObject.Invoke("set", param, value)); - private static T WrapAsType(JvmObjectReference reference) + internal static T WrapAsType(JvmObjectReference reference) { ConstructorInfo constructor = typeof(T) .GetConstructors(BindingFlags.NonPublic | BindingFlags.Instance) @@ -111,5 +111,6 @@ private static T WrapAsType(JvmObjectReference reference) return (T)constructor.Invoke(new object[] {reference}); } + } } diff --git a/src/csharp/Microsoft.Spark/Microsoft.Spark.csproj b/src/csharp/Microsoft.Spark/Microsoft.Spark.csproj index 2cddc5627..f284de8c6 100644 --- a/src/csharp/Microsoft.Spark/Microsoft.Spark.csproj +++ b/src/csharp/Microsoft.Spark/Microsoft.Spark.csproj @@ -38,10 +38,7 @@ - + From 60eb82f40ac37c553ca00a3ab4d0e404e4447dca Mon Sep 17 00:00:00 2001 From: GOEddieUK Date: Fri, 31 Jul 2020 19:52:23 +0100 Subject: [PATCH 07/20] undoing csproj change --- .ionide/symbolCache.db | Bin 28672 -> 0 bytes .../Microsoft.Spark/Microsoft.Spark.csproj | 5 ++++- 2 files changed, 4 insertions(+), 1 deletion(-) delete mode 100644 .ionide/symbolCache.db diff --git a/.ionide/symbolCache.db b/.ionide/symbolCache.db deleted file mode 100644 index 43e567d6d682d85dd32b3baebb0fdf61f67c1643..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28672 zcmeHPYiuJ|6}A(f3n*{Ao>?UI&RXX3biC7$u_ zru*Uw_|aboAq4!aPz(G7Dp3^)2^9$msX|362qA<70{*l}EvWneB<{TIImtK)+7qdu zu{?L~_%Yvi&pr3fz2}^JGgp@L0vB1UR7NW^3^wa~*(5A|iH8H;*B z&*Jr7uND(C74gzvnY~{#(YNt3Bw$FukbofpLjr~b3<($#FeG3|z>t6;0j&hSxNf$G zdV0*S4h!s^BA3}J-Ki9L<b^F{6=TjCuvK9>U*;l97n^=RUn$l~UNgYJCMRsfNA1?9Bl`LCS%K#Ngzf z;{QO@KD+;){!jcL`9JXd#`A*vLwDKru4~cx6X&wy2aZ|$+xDBbpW9ZfKeZmWylYu) z{ax!q%P(3^n}21#VfwY{Wv%+^=L?ZG(qpDCr$c=8^*P6`^IVl5<5tIVd0~tSzgigM z?z5uk`LPT6Y_-By)&wRae!(ne*4gR?lUBdKT&?7)Y>8Rp+k4w%>fhA!qkaWU!g8j& z9avyP?TH38h17hd$}v3E&T>vpABO?_g&-RIX#2Phe6h%7MQ!I9p4+7FTpy5icQ=}> z549iWIkuWzms8^H1xPCcSS58?UH(Q%WgSo}pSi&1%ghFqw{V?jb6g|G_h<$0h{v%C z-gd4n!}2^=x>Jxh}w>8SGqHC9v|D(^V*GlIoc1w|iaru~I2##}Q_|T1nBeK3-|27DujYN}U*Qnog;yWG zZr^}f59*n?f&gs=trhybzAhapG&7CzWFvl0k6CDOn7FsU92`wI{g3@Pu)FM&(n0byiegJk8tp$;dZ(^ zv=Y$fNsAXqBh!KP@Nsv!7c*PDz?GP*+?q1NVCFOrK}H;XiU+ZH0EsJTZI6;T-JAq- zjuS+DU_#0}R28`M+eFlgwc9Ox&3NKZQZlZ_NMqF!>tue`EenJqeO= z-(k%EWxt9n4P*W<`j}){81sMGVPVYwoyPqCY2B>G{GWCz|I_n-YoKm$sAt63{IKZ@ zCjVW()Axq2+xxcnw&!C{$^DLd+4ZLDu=DedKRby1-}XDUcWqa#zqMYm{LFH`_1mrC zmLIi@o8L1(L;w7AFA|##XrEvtfaXaX7#xVzi>ihYNYj*Mww$X`*YV|QzC^=M?s7b{ zR2E&Ad_Jr7vJDN1UN$i$u~P>{b1*gdEEdg`lZyRFw(CoE4YZL{_BddI=VPNxf)tFtiS1c{kwLRP_Bs2KJFK{M=Zfgp z5MzaS^-RLY01>>A4J9(PJCPk;3|-Gg3h=}8Y*2oI=KR!=4YAJvd^~4-*c;Zwp=d)e zwB3Zp8E>CHASQ8d{J&ySn^K6#J;CrWR!`-^; zqM`|6+mM`(61?aElrp4!0${VlSjO{EDzvau3op=cAg;PpUaK$*T(-!H0bn9Ea6!6~ zfK+Z2jZ}ANN{^JVURgcM@|@U<%-5<_8pe2m6F=O3P0ZtfS{ltsMe8cM8#S4aNHRO7 zP>{8>qXSDzJ6)YVwwmL`gM=7R(3d8$>Y$^w!>jYuqOlTbI-J)^}tn~k4R`# z%gp&{VwO;uNmcVHV%BjK=nOZ4Rh#YB_F$tnC4XE!;#3Ygq;Hbn}0)EjsT`7(7Hm(^n4Shvcww9bHjiGUTkb|JUFBEjjap;AiR+{ zRfi)Sw-Q%wk3G;24hEwfM_e&LA1|CP7zp+@8d-fr1qb>{%Ti8k6mY>C>QgR<x&4FUUSXg<^HJJmg z+yu~gPaO(^pg4n>Feuf`;H>7b0t(b?!3eIh&iXMNlE!-ryH6Vpc*fwWGc*$D;%gTqk&q-Cl6 z&AhPQ$ki?Yc)V6Ocw}&wR=2ebVDZ#{FgVz|M=$;yfW}jQD)4qec<>o;`)ASWbD(2j zKwGjUe(ny9$7ZuaBe${y!12mP!8_>_%6uN&A8%dEKw#jIev4{6q2zTjkIn%KC!$yb zu1H5@5XzU1b*Mvx8ey0CdUjSW00%Hm3Vam6}SWN^JQ4~XCiyMMBOimJ2yEVw`3RFM``6MP+8rh)j(l45I9&m?x{pdOk1GyTW(8~A=7nOb-8>4TyfiALgBG{yP< zfN9y}f7L(ad(C&&`$zA~ooi^`*Tu02K?}xl$2k0 zbAGCma`TY_FI()QQ7l(SGLY<5OiOdr+o8!4+e1*~+h_)dnql67B-dwfHZZUiz zzabvn1HdDMHf^Ll5)zb@WM2U=c;2LBe==J4CfT-n1+TU#`8q*c)H!K{K&_voHgaIsS6#Dmy)4GV_@uWLz6 zCQj|6Ygy;mRq^1efCpaJLU|n1bS;|z2Cr*L98HWhx|R)qg4eYq#uF1dLT%c$kd6vq zg6CvS^dv?aO!Sley-p$ur@`DqgVL}*9Lx0e7KVs&&S^rGFGR!5--h~O1_78cOIj<(HBye+O7eATP3F@vFj0ZfVApl|kbfCFCq0p*~C8rNO| z5O|G}`oPsFL8V8~HcD~+AGF&{{x#oUeXn>w_CEI7Jzw?M-QRS_UEg(`cE0b-IsWD- z*gvut>CcRpApt`Ih6D@=7!vqDl0cTI`}N6#m<}|fmUX}y7k4Wa4`L?jmVGkJ6*QPm zYp1g@6xEwkmnZ@8xJSUAea_QpR|;ZeGEFy$x=)|HN*vD)pBhe0C^NlncK8B;yY%7BE>k_JC|$KEu;nbLo^u3l z$A=f=Oa&b*Noz^uG)L4O_hgO6Lz)T9gtSOo1n!-O7mYD7>XTNG)BbMWD7*yKTnn{p2kcErn^;5QO|^~@dfjb=g`(vO!rQ!^N%?io_SZ? zsF<0ra;!kxqpY{^imti@RPfs8^z`%@jfgc_8rKLgN6`#2wD@X-UA_4A%qCGc(UU6~ zuc#A&3DvB(O|fV<3^&xn+pKdkFx9uS8&=()>Y`TKtVBwi1a7T|SN5q>Q|Fbs-R8Wi r5i_V}JR8*1*wjg2b^;fqlb8T7-kvp6;i)FU08O@N<|Vu8nsWLNN`{i9 diff --git a/src/csharp/Microsoft.Spark/Microsoft.Spark.csproj b/src/csharp/Microsoft.Spark/Microsoft.Spark.csproj index f284de8c6..2cddc5627 100644 --- a/src/csharp/Microsoft.Spark/Microsoft.Spark.csproj +++ b/src/csharp/Microsoft.Spark/Microsoft.Spark.csproj @@ -38,7 +38,10 @@ - + From ed36375561e3495a675f9ac14ab80f79f3fbb38d Mon Sep 17 00:00:00 2001 From: GOEddieUK Date: Fri, 31 Jul 2020 19:55:49 +0100 Subject: [PATCH 08/20] member doesnt need to be internal --- src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs b/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs index 0895dace1..8446b9f4e 100644 --- a/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs +++ b/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs @@ -98,7 +98,7 @@ public Param.Param GetParam(string paramName) => public T Set(Param.Param param, object value) => WrapAsType((JvmObjectReference)_jvmObject.Invoke("set", param, value)); - internal static T WrapAsType(JvmObjectReference reference) + private static T WrapAsType(JvmObjectReference reference) { ConstructorInfo constructor = typeof(T) .GetConstructors(BindingFlags.NonPublic | BindingFlags.Instance) From c7baf7231914b10300175e67158b604d646b97d4 Mon Sep 17 00:00:00 2001 From: GOEddieUK Date: Fri, 31 Jul 2020 19:56:29 +0100 Subject: [PATCH 09/20] too many lines --- src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs b/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs index 8446b9f4e..9ccd64d5b 100644 --- a/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs +++ b/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs @@ -106,11 +106,10 @@ private static T WrapAsType(JvmObjectReference reference) { ParameterInfo[] parameters = c.GetParameters(); return (parameters.Length == 1) && - (parameters[0].ParameterType == typeof(JvmObjectReference)); + (parameters[0].ParameterType == typeof(JvmObjectReference)); }); return (T)constructor.Invoke(new object[] {reference}); } - } } From d13303ccaeb691691c4d294d96e0995f3597becb Mon Sep 17 00:00:00 2001 From: GOEddieUK Date: Fri, 31 Jul 2020 20:01:07 +0100 Subject: [PATCH 10/20] removing whitespace change --- src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs b/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs index 9ccd64d5b..326268a5e 100644 --- a/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs +++ b/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs @@ -105,7 +105,7 @@ private static T WrapAsType(JvmObjectReference reference) .Single(c => { ParameterInfo[] parameters = c.GetParameters(); - return (parameters.Length == 1) && + return (parameters.Length == 1) && (parameters[0].ParameterType == typeof(JvmObjectReference)); }); From f5b477c72158599b1c6552c7eb1af20edfab7779 Mon Sep 17 00:00:00 2001 From: GOEddieUK Date: Fri, 31 Jul 2020 20:01:57 +0100 Subject: [PATCH 11/20] removing whitespace change --- src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs b/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs index 326268a5e..9ccd64d5b 100644 --- a/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs +++ b/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs @@ -105,7 +105,7 @@ private static T WrapAsType(JvmObjectReference reference) .Single(c => { ParameterInfo[] parameters = c.GetParameters(); - return (parameters.Length == 1) && + return (parameters.Length == 1) && (parameters[0].ParameterType == typeof(JvmObjectReference)); }); From 73db52b400637585b2216f44aac616828800b9d2 Mon Sep 17 00:00:00 2001 From: GOEddieUK Date: Fri, 31 Jul 2020 20:06:12 +0100 Subject: [PATCH 12/20] ionide --- .ionide/symbolCache.db | Bin 0 -> 28672 bytes .../Microsoft.Spark/ML/Feature/FeatureBase.cs | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 .ionide/symbolCache.db diff --git a/.ionide/symbolCache.db b/.ionide/symbolCache.db new file mode 100644 index 0000000000000000000000000000000000000000..43e567d6d682d85dd32b3baebb0fdf61f67c1643 GIT binary patch literal 28672 zcmeHPYiuJ|6}A(f3n*{Ao>?UI&RXX3biC7$u_ zru*Uw_|aboAq4!aPz(G7Dp3^)2^9$msX|362qA<70{*l}EvWneB<{TIImtK)+7qdu zu{?L~_%Yvi&pr3fz2}^JGgp@L0vB1UR7NW^3^wa~*(5A|iH8H;*B z&*Jr7uND(C74gzvnY~{#(YNt3Bw$FukbofpLjr~b3<($#FeG3|z>t6;0j&hSxNf$G zdV0*S4h!s^BA3}J-Ki9L<b^F{6=TjCuvK9>U*;l97n^=RUn$l~UNgYJCMRsfNA1?9Bl`LCS%K#Ngzf z;{QO@KD+;){!jcL`9JXd#`A*vLwDKru4~cx6X&wy2aZ|$+xDBbpW9ZfKeZmWylYu) z{ax!q%P(3^n}21#VfwY{Wv%+^=L?ZG(qpDCr$c=8^*P6`^IVl5<5tIVd0~tSzgigM z?z5uk`LPT6Y_-By)&wRae!(ne*4gR?lUBdKT&?7)Y>8Rp+k4w%>fhA!qkaWU!g8j& z9avyP?TH38h17hd$}v3E&T>vpABO?_g&-RIX#2Phe6h%7MQ!I9p4+7FTpy5icQ=}> z549iWIkuWzms8^H1xPCcSS58?UH(Q%WgSo}pSi&1%ghFqw{V?jb6g|G_h<$0h{v%C z-gd4n!}2^=x>Jxh}w>8SGqHC9v|D(^V*GlIoc1w|iaru~I2##}Q_|T1nBeK3-|27DujYN}U*Qnog;yWG zZr^}f59*n?f&gs=trhybzAhapG&7CzWFvl0k6CDOn7FsU92`wI{g3@Pu)FM&(n0byiegJk8tp$;dZ(^ zv=Y$fNsAXqBh!KP@Nsv!7c*PDz?GP*+?q1NVCFOrK}H;XiU+ZH0EsJTZI6;T-JAq- zjuS+DU_#0}R28`M+eFlgwc9Ox&3NKZQZlZ_NMqF!>tue`EenJqeO= z-(k%EWxt9n4P*W<`j}){81sMGVPVYwoyPqCY2B>G{GWCz|I_n-YoKm$sAt63{IKZ@ zCjVW()Axq2+xxcnw&!C{$^DLd+4ZLDu=DedKRby1-}XDUcWqa#zqMYm{LFH`_1mrC zmLIi@o8L1(L;w7AFA|##XrEvtfaXaX7#xVzi>ihYNYj*Mww$X`*YV|QzC^=M?s7b{ zR2E&Ad_Jr7vJDN1UN$i$u~P>{b1*gdEEdg`lZyRFw(CoE4YZL{_BddI=VPNxf)tFtiS1c{kwLRP_Bs2KJFK{M=Zfgp z5MzaS^-RLY01>>A4J9(PJCPk;3|-Gg3h=}8Y*2oI=KR!=4YAJvd^~4-*c;Zwp=d)e zwB3Zp8E>CHASQ8d{J&ySn^K6#J;CrWR!`-^; zqM`|6+mM`(61?aElrp4!0${VlSjO{EDzvau3op=cAg;PpUaK$*T(-!H0bn9Ea6!6~ zfK+Z2jZ}ANN{^JVURgcM@|@U<%-5<_8pe2m6F=O3P0ZtfS{ltsMe8cM8#S4aNHRO7 zP>{8>qXSDzJ6)YVwwmL`gM=7R(3d8$>Y$^w!>jYuqOlTbI-J)^}tn~k4R`# z%gp&{VwO;uNmcVHV%BjK=nOZ4Rh#YB_F$tnC4XE!;#3Ygq;Hbn}0)EjsT`7(7Hm(^n4Shvcww9bHjiGUTkb|JUFBEjjap;AiR+{ zRfi)Sw-Q%wk3G;24hEwfM_e&LA1|CP7zp+@8d-fr1qb>{%Ti8k6mY>C>QgR<x&4FUUSXg<^HJJmg z+yu~gPaO(^pg4n>Feuf`;H>7b0t(b?!3eIh&iXMNlE!-ryH6Vpc*fwWGc*$D;%gTqk&q-Cl6 z&AhPQ$ki?Yc)V6Ocw}&wR=2ebVDZ#{FgVz|M=$;yfW}jQD)4qec<>o;`)ASWbD(2j zKwGjUe(ny9$7ZuaBe${y!12mP!8_>_%6uN&A8%dEKw#jIev4{6q2zTjkIn%KC!$yb zu1H5@5XzU1b*Mvx8ey0CdUjSW00%Hm3Vam6}SWN^JQ4~XCiyMMBOimJ2yEVw`3RFM``6MP+8rh)j(l45I9&m?x{pdOk1GyTW(8~A=7nOb-8>4TyfiALgBG{yP< zfN9y}f7L(ad(C&&`$zA~ooi^`*Tu02K?}xl$2k0 zbAGCma`TY_FI()QQ7l(SGLY<5OiOdr+o8!4+e1*~+h_)dnql67B-dwfHZZUiz zzabvn1HdDMHf^Ll5)zb@WM2U=c;2LBe==J4CfT-n1+TU#`8q*c)H!K{K&_voHgaIsS6#Dmy)4GV_@uWLz6 zCQj|6Ygy;mRq^1efCpaJLU|n1bS;|z2Cr*L98HWhx|R)qg4eYq#uF1dLT%c$kd6vq zg6CvS^dv?aO!Sley-p$ur@`DqgVL}*9Lx0e7KVs&&S^rGFGR!5--h~O1_78cOIj<(HBye+O7eATP3F@vFj0ZfVApl|kbfCFCq0p*~C8rNO| z5O|G}`oPsFL8V8~HcD~+AGF&{{x#oUeXn>w_CEI7Jzw?M-QRS_UEg(`cE0b-IsWD- z*gvut>CcRpApt`Ih6D@=7!vqDl0cTI`}N6#m<}|fmUX}y7k4Wa4`L?jmVGkJ6*QPm zYp1g@6xEwkmnZ@8xJSUAea_QpR|;ZeGEFy$x=)|HN*vD)pBhe0C^NlncK8B;yY%7BE>k_JC|$KEu;nbLo^u3l z$A=f=Oa&b*Noz^uG)L4O_hgO6Lz)T9gtSOo1n!-O7mYD7>XTNG)BbMWD7*yKTnn{p2kcErn^;5QO|^~@dfjb=g`(vO!rQ!^N%?io_SZ? zsF<0ra;!kxqpY{^imti@RPfs8^z`%@jfgc_8rKLgN6`#2wD@X-UA_4A%qCGc(UU6~ zuc#A&3DvB(O|fV<3^&xn+pKdkFx9uS8&=()>Y`TKtVBwi1a7T|SN5q>Q|Fbs-R8Wi r5i_V}JR8*1*wjg2b^;fqlb8T7-kvp6;i)FU08O@N<|Vu8nsWLNN`{i9 literal 0 HcmV?d00001 diff --git a/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs b/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs index 9ccd64d5b..326268a5e 100644 --- a/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs +++ b/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs @@ -105,7 +105,7 @@ private static T WrapAsType(JvmObjectReference reference) .Single(c => { ParameterInfo[] parameters = c.GetParameters(); - return (parameters.Length == 1) && + return (parameters.Length == 1) && (parameters[0].ParameterType == typeof(JvmObjectReference)); }); From 8e1685cd270657c5e7a6769e732bf85d5ae6cb2e Mon Sep 17 00:00:00 2001 From: Niharika Dutta Date: Thu, 13 Aug 2020 12:59:34 -0700 Subject: [PATCH 13/20] Revert "Merge branch 'master' into ml/countvectorizer" This reverts commit a766146f56014ccae4118b35495b84da588af94f, reversing changes made to 73db52b400637585b2216f44aac616828800b9d2. Reverting countvectorizer changes --- .gitignore | 3 --- .ionide/symbolCache.db | Bin 0 -> 28672 bytes .../Processor/BroadcastVariableProcessor.cs | 3 +-- 3 files changed, 1 insertion(+), 5 deletions(-) create mode 100644 .ionide/symbolCache.db diff --git a/.gitignore b/.gitignore index faada9c8a..251cfa7e2 100644 --- a/.gitignore +++ b/.gitignore @@ -367,6 +367,3 @@ hs_err_pid* # The target folder contains the output of building **/target/** - -# F# vs code -.ionide/ diff --git a/.ionide/symbolCache.db b/.ionide/symbolCache.db new file mode 100644 index 0000000000000000000000000000000000000000..43e567d6d682d85dd32b3baebb0fdf61f67c1643 GIT binary patch literal 28672 zcmeHPYiuJ|6}A(f3n*{Ao>?UI&RXX3biC7$u_ zru*Uw_|aboAq4!aPz(G7Dp3^)2^9$msX|362qA<70{*l}EvWneB<{TIImtK)+7qdu zu{?L~_%Yvi&pr3fz2}^JGgp@L0vB1UR7NW^3^wa~*(5A|iH8H;*B z&*Jr7uND(C74gzvnY~{#(YNt3Bw$FukbofpLjr~b3<($#FeG3|z>t6;0j&hSxNf$G zdV0*S4h!s^BA3}J-Ki9L<b^F{6=TjCuvK9>U*;l97n^=RUn$l~UNgYJCMRsfNA1?9Bl`LCS%K#Ngzf z;{QO@KD+;){!jcL`9JXd#`A*vLwDKru4~cx6X&wy2aZ|$+xDBbpW9ZfKeZmWylYu) z{ax!q%P(3^n}21#VfwY{Wv%+^=L?ZG(qpDCr$c=8^*P6`^IVl5<5tIVd0~tSzgigM z?z5uk`LPT6Y_-By)&wRae!(ne*4gR?lUBdKT&?7)Y>8Rp+k4w%>fhA!qkaWU!g8j& z9avyP?TH38h17hd$}v3E&T>vpABO?_g&-RIX#2Phe6h%7MQ!I9p4+7FTpy5icQ=}> z549iWIkuWzms8^H1xPCcSS58?UH(Q%WgSo}pSi&1%ghFqw{V?jb6g|G_h<$0h{v%C z-gd4n!}2^=x>Jxh}w>8SGqHC9v|D(^V*GlIoc1w|iaru~I2##}Q_|T1nBeK3-|27DujYN}U*Qnog;yWG zZr^}f59*n?f&gs=trhybzAhapG&7CzWFvl0k6CDOn7FsU92`wI{g3@Pu)FM&(n0byiegJk8tp$;dZ(^ zv=Y$fNsAXqBh!KP@Nsv!7c*PDz?GP*+?q1NVCFOrK}H;XiU+ZH0EsJTZI6;T-JAq- zjuS+DU_#0}R28`M+eFlgwc9Ox&3NKZQZlZ_NMqF!>tue`EenJqeO= z-(k%EWxt9n4P*W<`j}){81sMGVPVYwoyPqCY2B>G{GWCz|I_n-YoKm$sAt63{IKZ@ zCjVW()Axq2+xxcnw&!C{$^DLd+4ZLDu=DedKRby1-}XDUcWqa#zqMYm{LFH`_1mrC zmLIi@o8L1(L;w7AFA|##XrEvtfaXaX7#xVzi>ihYNYj*Mww$X`*YV|QzC^=M?s7b{ zR2E&Ad_Jr7vJDN1UN$i$u~P>{b1*gdEEdg`lZyRFw(CoE4YZL{_BddI=VPNxf)tFtiS1c{kwLRP_Bs2KJFK{M=Zfgp z5MzaS^-RLY01>>A4J9(PJCPk;3|-Gg3h=}8Y*2oI=KR!=4YAJvd^~4-*c;Zwp=d)e zwB3Zp8E>CHASQ8d{J&ySn^K6#J;CrWR!`-^; zqM`|6+mM`(61?aElrp4!0${VlSjO{EDzvau3op=cAg;PpUaK$*T(-!H0bn9Ea6!6~ zfK+Z2jZ}ANN{^JVURgcM@|@U<%-5<_8pe2m6F=O3P0ZtfS{ltsMe8cM8#S4aNHRO7 zP>{8>qXSDzJ6)YVwwmL`gM=7R(3d8$>Y$^w!>jYuqOlTbI-J)^}tn~k4R`# z%gp&{VwO;uNmcVHV%BjK=nOZ4Rh#YB_F$tnC4XE!;#3Ygq;Hbn}0)EjsT`7(7Hm(^n4Shvcww9bHjiGUTkb|JUFBEjjap;AiR+{ zRfi)Sw-Q%wk3G;24hEwfM_e&LA1|CP7zp+@8d-fr1qb>{%Ti8k6mY>C>QgR<x&4FUUSXg<^HJJmg z+yu~gPaO(^pg4n>Feuf`;H>7b0t(b?!3eIh&iXMNlE!-ryH6Vpc*fwWGc*$D;%gTqk&q-Cl6 z&AhPQ$ki?Yc)V6Ocw}&wR=2ebVDZ#{FgVz|M=$;yfW}jQD)4qec<>o;`)ASWbD(2j zKwGjUe(ny9$7ZuaBe${y!12mP!8_>_%6uN&A8%dEKw#jIev4{6q2zTjkIn%KC!$yb zu1H5@5XzU1b*Mvx8ey0CdUjSW00%Hm3Vam6}SWN^JQ4~XCiyMMBOimJ2yEVw`3RFM``6MP+8rh)j(l45I9&m?x{pdOk1GyTW(8~A=7nOb-8>4TyfiALgBG{yP< zfN9y}f7L(ad(C&&`$zA~ooi^`*Tu02K?}xl$2k0 zbAGCma`TY_FI()QQ7l(SGLY<5OiOdr+o8!4+e1*~+h_)dnql67B-dwfHZZUiz zzabvn1HdDMHf^Ll5)zb@WM2U=c;2LBe==J4CfT-n1+TU#`8q*c)H!K{K&_voHgaIsS6#Dmy)4GV_@uWLz6 zCQj|6Ygy;mRq^1efCpaJLU|n1bS;|z2Cr*L98HWhx|R)qg4eYq#uF1dLT%c$kd6vq zg6CvS^dv?aO!Sley-p$ur@`DqgVL}*9Lx0e7KVs&&S^rGFGR!5--h~O1_78cOIj<(HBye+O7eATP3F@vFj0ZfVApl|kbfCFCq0p*~C8rNO| z5O|G}`oPsFL8V8~HcD~+AGF&{{x#oUeXn>w_CEI7Jzw?M-QRS_UEg(`cE0b-IsWD- z*gvut>CcRpApt`Ih6D@=7!vqDl0cTI`}N6#m<}|fmUX}y7k4Wa4`L?jmVGkJ6*QPm zYp1g@6xEwkmnZ@8xJSUAea_QpR|;ZeGEFy$x=)|HN*vD)pBhe0C^NlncK8B;yY%7BE>k_JC|$KEu;nbLo^u3l z$A=f=Oa&b*Noz^uG)L4O_hgO6Lz)T9gtSOo1n!-O7mYD7>XTNG)BbMWD7*yKTnn{p2kcErn^;5QO|^~@dfjb=g`(vO!rQ!^N%?io_SZ? zsF<0ra;!kxqpY{^imti@RPfs8^z`%@jfgc_8rKLgN6`#2wD@X-UA_4A%qCGc(UU6~ zuc#A&3DvB(O|fV<3^&xn+pKdkFx9uS8&=()>Y`TKtVBwi1a7T|SN5q>Q|Fbs-R8Wi r5i_V}JR8*1*wjg2b^;fqlb8T7-kvp6;i)FU08O@N<|Vu8nsWLNN`{i9 literal 0 HcmV?d00001 diff --git a/src/csharp/Microsoft.Spark.Worker/Processor/BroadcastVariableProcessor.cs b/src/csharp/Microsoft.Spark.Worker/Processor/BroadcastVariableProcessor.cs index bf8f48ed8..41c817d02 100644 --- a/src/csharp/Microsoft.Spark.Worker/Processor/BroadcastVariableProcessor.cs +++ b/src/csharp/Microsoft.Spark.Worker/Processor/BroadcastVariableProcessor.cs @@ -54,8 +54,7 @@ internal BroadcastVariables Process(Stream stream) else { string path = SerDe.ReadString(stream); - using FileStream fStream = - File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read); + using FileStream fStream = File.Open(path, FileMode.Open, FileAccess.Read); object value = formatter.Deserialize(fStream); BroadcastRegistry.Add(bid, value); } From 255515eecbd6cb8e7919fbd2b857d99e335c66d2 Mon Sep 17 00:00:00 2001 From: Niharika Dutta Date: Thu, 13 Aug 2020 13:04:05 -0700 Subject: [PATCH 14/20] Revert "Merge branch 'ml/countvectorizer' of https://github.com/GoEddie/spark" This reverts commit ad6bcede69de012c22178825e76c6b175c770b8f, reversing changes made to 4c5d502a9f56e79ea071b12d2a49dced3873dea8. reverting countvectorizer changes -2 --- .../ML/Feature/CountVectorizerModelTests.cs | 73 ------- .../ML/Feature/CountVectorizerTests.cs | 76 ------- .../ML/Feature/CountVectorizer.cs | 197 ------------------ .../ML/Feature/CountVectorizerModel.cs | 170 --------------- .../Microsoft.Spark/ML/Feature/FeatureBase.cs | 4 +- 5 files changed, 2 insertions(+), 518 deletions(-) delete mode 100644 src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerModelTests.cs delete mode 100644 src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerTests.cs delete mode 100644 src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs delete mode 100644 src/csharp/Microsoft.Spark/ML/Feature/CountVectorizerModel.cs diff --git a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerModelTests.cs b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerModelTests.cs deleted file mode 100644 index 3c3132dd9..000000000 --- a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerModelTests.cs +++ /dev/null @@ -1,73 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.IO; -using Microsoft.Spark.ML.Feature; -using Microsoft.Spark.Sql; -using Microsoft.Spark.UnitTest.TestUtils; -using Xunit; - -namespace Microsoft.Spark.E2ETest.IpcTests.ML.Feature -{ - [Collection("Spark E2E Tests")] - public class CountVectorizerModelTests - { - private readonly SparkSession _spark; - - public CountVectorizerModelTests(SparkFixture fixture) - { - _spark = fixture.Spark; - } - - [Fact] - public void Test_CountVectorizerModel() - { - DataFrame input = _spark.Sql("SELECT array('hello', 'I', 'AM', 'a', 'string', 'TO', " + - "'TOKENIZE') as input from range(100)"); - - const string inputColumn = "input"; - const string outputColumn = "output"; - const double minTf = 10.0; - const bool binary = false; - - List vocabulary = new List() - { - "hello", - "I", - "AM", - "TO", - "TOKENIZE" - }; - - var countVectorizerModel = new CountVectorizerModel(vocabulary); - - Assert.IsType(new CountVectorizerModel("my-uid", vocabulary)); - - countVectorizerModel = countVectorizerModel - .SetInputCol(inputColumn) - .SetOutputCol(outputColumn) - .SetMinTF(minTf) - .SetBinary(binary); - - Assert.Equal(inputColumn, countVectorizerModel.GetInputCol()); - Assert.Equal(outputColumn, countVectorizerModel.GetOutputCol()); - Assert.Equal(minTf, countVectorizerModel.GetMinTF()); - Assert.Equal(binary, countVectorizerModel.GetBinary()); - using (var tempDirectory = new TemporaryDirectory()) - { - string savePath = Path.Join(tempDirectory.Path, "countVectorizerModel"); - countVectorizerModel.Save(savePath); - - CountVectorizerModel loadedModel = CountVectorizerModel.Load(savePath); - Assert.Equal(countVectorizerModel.Uid(), loadedModel.Uid()); - } - - Assert.IsType(countVectorizerModel.GetVocabSize()); - Assert.NotEmpty(countVectorizerModel.ExplainParams()); - Assert.NotEmpty(countVectorizerModel.ToString()); - } - } -} diff --git a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerTests.cs b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerTests.cs deleted file mode 100644 index 95b9bc504..000000000 --- a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerTests.cs +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.IO; -using Microsoft.Spark.E2ETest.Utils; -using Microsoft.Spark.ML.Feature; -using Microsoft.Spark.Sql; -using Microsoft.Spark.UnitTest.TestUtils; -using Xunit; - -namespace Microsoft.Spark.E2ETest.IpcTests.ML.Feature -{ - [Collection("Spark E2E Tests")] - public class CountVectorizerTests - { - private readonly SparkSession _spark; - - public CountVectorizerTests(SparkFixture fixture) - { - _spark = fixture.Spark; - } - - [Fact] - public void Test_CountVectorizer() - { - DataFrame input = _spark.Sql("SELECT array('hello', 'I', 'AM', 'a', 'string', 'TO', " + - "'TOKENIZE') as input from range(100)"); - - const string inputColumn = "input"; - const string outputColumn = "output"; - const double minDf = 1; - const double minTf = 10; - const int vocabSize = 10000; - const bool binary = false; - - var countVectorizer = new CountVectorizer(); - - countVectorizer - .SetInputCol(inputColumn) - .SetOutputCol(outputColumn) - .SetMinDF(minDf) - .SetMinTF(minTf) - .SetVocabSize(vocabSize); - - Assert.IsType(countVectorizer.Fit(input)); - Assert.Equal(inputColumn, countVectorizer.GetInputCol()); - Assert.Equal(outputColumn, countVectorizer.GetOutputCol()); - Assert.Equal(minDf, countVectorizer.GetMinDF()); - Assert.Equal(minTf, countVectorizer.GetMinTF()); - Assert.Equal(vocabSize, countVectorizer.GetVocabSize()); - Assert.Equal(binary, countVectorizer.GetBinary()); - - using (var tempDirectory = new TemporaryDirectory()) - { - string savePath = Path.Join(tempDirectory.Path, "countVectorizer"); - countVectorizer.Save(savePath); - - CountVectorizer loadedVectorizer = CountVectorizer.Load(savePath); - Assert.Equal(countVectorizer.Uid(), loadedVectorizer.Uid()); - } - - Assert.NotEmpty(countVectorizer.ExplainParams()); - Assert.NotEmpty(countVectorizer.ToString()); - } - - [SkipIfSparkVersionIsLessThan(Versions.V2_4_0)] - public void CountVectorizer_MaxDF() - { - const double maxDf = 100; - CountVectorizer countVectorizer = new CountVectorizer().SetMaxDF(maxDf); - Assert.Equal(maxDf, countVectorizer.GetMaxDF()); - } - } -} diff --git a/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs b/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs deleted file mode 100644 index 5689e19fd..000000000 --- a/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizer.cs +++ /dev/null @@ -1,197 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.Spark.Interop; -using Microsoft.Spark.Interop.Ipc; -using Microsoft.Spark.Sql; - -namespace Microsoft.Spark.ML.Feature -{ - public class CountVectorizer : FeatureBase, IJvmObjectReferenceProvider - { - private static readonly string s_countVectorizerClassName = - "org.apache.spark.ml.feature.CountVectorizer"; - - /// - /// Create a without any parameters - /// - public CountVectorizer() : base(s_countVectorizerClassName) - { - } - - /// - /// Create a with a UID that is used to give the - /// a unique ID - /// - /// An immutable unique ID for the object and its derivatives. - public CountVectorizer(string uid) : base(s_countVectorizerClassName, uid) - { - } - - internal CountVectorizer(JvmObjectReference jvmObject) : base(jvmObject) - { - } - - JvmObjectReference IJvmObjectReferenceProvider.Reference => _jvmObject; - - /// Fits a model to the input data. - /// The to fit the model to. - /// - public CountVectorizerModel Fit(DataFrame dataFrame) => - new CountVectorizerModel((JvmObjectReference)_jvmObject.Invoke("fit", dataFrame)); - - /// - /// Loads the that was previously saved using Save - /// - /// - /// The path the previous was saved to - /// - /// New object - public static CountVectorizer Load(string path) => - WrapAsCountVectorizer((JvmObjectReference) - SparkEnvironment.JvmBridge.CallStaticJavaMethod( - s_countVectorizerClassName,"load", path)); - - /// - /// Gets the binary toggle to control the output vector values. If True, all nonzero counts - /// (after minTF filter applied) are set to 1. This is useful for discrete probabilistic - /// models that model binary events rather than integer counts. Default: false - /// - /// boolean - public bool GetBinary() => (bool)_jvmObject.Invoke("getBinary"); - - /// - /// Sets the binary toggle to control the output vector values. If True, all nonzero counts - /// (after minTF filter applied) are set to 1. This is useful for discrete probabilistic - /// models that model binary events rather than integer counts. Default: false - /// - /// Turn the binary toggle on or off - /// with the new binary toggle value set - public CountVectorizer SetBinary(bool value) => - WrapAsCountVectorizer((JvmObjectReference)_jvmObject.Invoke("setBinary", value)); - - /// - /// Gets the column that the should read from and convert - /// into buckets. This would have been set by SetInputCol - /// - /// string, the input column - public string GetInputCol() => _jvmObject.Invoke("getInputCol") as string; - - /// - /// Sets the column that the should read from. - /// - /// The name of the column to as the source. - /// with the input column set - public CountVectorizer SetInputCol(string value) => - WrapAsCountVectorizer((JvmObjectReference)_jvmObject.Invoke("setInputCol", value)); - - /// - /// The will create a new column in the DataFrame, this is - /// the name of the new column. - /// - /// The name of the output column. - public string GetOutputCol() => _jvmObject.Invoke("getOutputCol") as string; - - /// - /// The will create a new column in the DataFrame, this - /// is the name of the new column. - /// - /// The name of the output column which will be created. - /// New with the output column set - public CountVectorizer SetOutputCol(string value) => - WrapAsCountVectorizer((JvmObjectReference)_jvmObject.Invoke("setOutputCol", value)); - - /// - /// Gets the maximum number of different documents a term could appear in to be included in - /// the vocabulary. A term that appears more than the threshold will be ignored. If this is - /// an integer greater than or equal to 1, this specifies the maximum number of documents - /// the term could appear in; if this is a double in [0,1), then this specifies the maximum - /// fraction of documents the term could appear in. - /// - /// The maximum document term frequency - [Since(Versions.V2_4_0)] - public double GetMaxDF() => (double)_jvmObject.Invoke("getMaxDF"); - - /// - /// Sets the maximum number of different documents a term could appear in to be included in - /// the vocabulary. A term that appears more than the threshold will be ignored. If this is - /// an integer greater than or equal to 1, this specifies the maximum number of documents - /// the term could appear in; if this is a double in [0,1), then this specifies the maximum - /// fraction of documents the term could appear in. - /// - /// The maximum document term frequency - /// New with the max df value set - [Since(Versions.V2_4_0)] - public CountVectorizer SetMaxDF(double value) => - WrapAsCountVectorizer((JvmObjectReference)_jvmObject.Invoke("setMaxDF", value)); - - /// - /// Gets the minimum number of different documents a term must appear in to be included in - /// the vocabulary. If this is an integer greater than or equal to 1, this specifies the - /// number of documents the term must appear in; if this is a double in [0,1), then this - /// specifies the fraction of documents. - /// - /// The minimum document term frequency - public double GetMinDF() => (double)_jvmObject.Invoke("getMinDF"); - - /// - /// Sets the minimum number of different documents a term must appear in to be included in - /// the vocabulary. If this is an integer greater than or equal to 1, this specifies the - /// number of documents the term must appear in; if this is a double in [0,1), then this - /// specifies the fraction of documents. - /// - /// The minimum document term frequency - /// New with the min df value set - public CountVectorizer SetMinDF(double value) => - WrapAsCountVectorizer((JvmObjectReference)_jvmObject.Invoke("setMinDF", value)); - - /// - /// Filter to ignore rare words in a document. For each document, terms with - /// frequency/count less than the given threshold are ignored. If this is an integer - /// greater than or equal to 1, then this specifies a count (of times the term must appear - /// in the document); if this is a double in [0,1), then this specifies a fraction (out of - /// the document's token count). - /// - /// Note that the parameter is only used in transform of CountVectorizerModel and does not - /// affect fitting. - /// - /// Minimum term frequency - public double GetMinTF() => (double)_jvmObject.Invoke("getMinTF"); - - /// - /// Filter to ignore rare words in a document. For each document, terms with - /// frequency/count less than the given threshold are ignored. If this is an integer - /// greater than or equal to 1, then this specifies a count (of times the term must appear - /// in the document); if this is a double in [0,1), then this specifies a fraction (out of - /// the document's token count). - /// - /// Note that the parameter is only used in transform of CountVectorizerModel and does not - /// affect fitting. - /// - /// Minimum term frequency - /// New with the min term frequency set - public CountVectorizer SetMinTF(double value) => - WrapAsCountVectorizer((JvmObjectReference)_jvmObject.Invoke("setMinTF", value)); - - /// - /// Gets the max size of the vocabulary. CountVectorizer will build a vocabulary that only - /// considers the top vocabSize terms ordered by term frequency across the corpus. - /// - /// The max size of the vocabulary - public int GetVocabSize() => (int)_jvmObject.Invoke("getVocabSize"); - - /// - /// Sets the max size of the vocabulary. will build a - /// vocabulary that only considers the top vocabSize terms ordered by term frequency across - /// the corpus. - /// - /// The max vocabulary size - /// with the max vocab value set - public CountVectorizer SetVocabSize(int value) => - WrapAsCountVectorizer(_jvmObject.Invoke("setVocabSize", value)); - - private static CountVectorizer WrapAsCountVectorizer(object obj) => - new CountVectorizer((JvmObjectReference)obj); - } -} diff --git a/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizerModel.cs b/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizerModel.cs deleted file mode 100644 index 52bbd72c3..000000000 --- a/src/csharp/Microsoft.Spark/ML/Feature/CountVectorizerModel.cs +++ /dev/null @@ -1,170 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Collections.Generic; -using Microsoft.Spark.Interop; -using Microsoft.Spark.Interop.Ipc; - -namespace Microsoft.Spark.ML.Feature -{ - public class CountVectorizerModel : FeatureBase - , IJvmObjectReferenceProvider - { - private static readonly string s_countVectorizerModelClassName = - "org.apache.spark.ml.feature.CountVectorizerModel"; - - /// - /// Create a without any parameters - /// - /// The vocabulary to use - public CountVectorizerModel(List vocabulary) : - this(SparkEnvironment.JvmBridge.CallConstructor( - s_countVectorizerModelClassName, vocabulary)) - { - } - - /// - /// Create a with a UID that is used to give the - /// a unique ID - /// - /// An immutable unique ID for the object and its derivatives. - /// The vocabulary to use - public CountVectorizerModel(string uid, List vocabulary) : - this(SparkEnvironment.JvmBridge.CallConstructor( - s_countVectorizerModelClassName, uid, vocabulary)) - { - } - - internal CountVectorizerModel(JvmObjectReference jvmObject) : base(jvmObject) - { - } - - JvmObjectReference IJvmObjectReferenceProvider.Reference => _jvmObject; - - /// - /// Loads the that was previously saved using Save - /// - /// - /// The path the previous was saved to - /// - /// New object - public static CountVectorizerModel Load(string path) => - WrapAsCountVectorizerModel((JvmObjectReference) - SparkEnvironment.JvmBridge.CallStaticJavaMethod( - s_countVectorizerModelClassName,"load", path)); - - /// - /// Gets the binary toggle to control the output vector values. If True, all nonzero counts - /// (after minTF filter applied) are set to 1. This is useful for discrete probabilistic - /// models that model binary events rather than integer counts. Default: false - /// - /// boolean - public bool GetBinary() => (bool)_jvmObject.Invoke("getBinary"); - - /// - /// Sets the binary toggle to control the output vector values. If True, all nonzero counts - /// (after minTF filter applied) are set to 1. This is useful for discrete probabilistic - /// models that model binary events rather than integer counts. Default: false - /// - /// Turn the binary toggle on or off - /// - /// with the new binary toggle value set - /// - public CountVectorizerModel SetBinary(bool value) => - WrapAsCountVectorizerModel((JvmObjectReference)_jvmObject.Invoke("setBinary", value)); - - /// - /// Gets the column that the should read from and - /// convert into buckets. This would have been set by SetInputCol - /// - /// string, the input column - public string GetInputCol() => _jvmObject.Invoke("getInputCol") as string; - - /// - /// Sets the column that the should read from. - /// - /// The name of the column to as the source. - /// with the input column set - public CountVectorizerModel SetInputCol(string value) => - WrapAsCountVectorizerModel( - (JvmObjectReference)_jvmObject.Invoke("setInputCol", value)); - - /// - /// The will create a new column in the DataFrame, this - /// is the name of the new column. - /// - /// The name of the output column. - public string GetOutputCol() => _jvmObject.Invoke("getOutputCol") as string; - - /// - /// The will create a new column in the DataFrame, - /// this is the name of the new column. - /// - /// The name of the output column which will be created. - /// New with the output column set - public CountVectorizerModel SetOutputCol(string value) => - WrapAsCountVectorizerModel( - (JvmObjectReference)_jvmObject.Invoke("setOutputCol", value)); - - /// - /// Gets the maximum number of different documents a term could appear in to be included in - /// the vocabulary. A term that appears more than the threshold will be ignored. If this is - /// an integer greater than or equal to 1, this specifies the maximum number of documents - /// the term could appear in; if this is a double in [0,1), then this specifies the maximum - /// fraction of documents the term could appear in. - /// - /// The maximum document term frequency - public double GetMaxDF() => (double)_jvmObject.Invoke("getMaxDF"); - - /// - /// Gets the minimum number of different documents a term must appear in to be included in - /// the vocabulary. If this is an integer greater than or equal to 1, this specifies the - /// number of documents the term must appear in; if this is a double in [0,1), then this - /// specifies the fraction of documents. - /// - /// The minimum document term frequency - public double GetMinDF() => (double)_jvmObject.Invoke("getMinDF"); - - /// - /// Filter to ignore rare words in a document. For each document, terms with - /// frequency/count less than the given threshold are ignored. If this is an integer - /// greater than or equal to 1, then this specifies a count (of times the term must appear - /// in the document); if this is a double in [0,1), then this specifies a fraction (out of - /// the document's token count). - /// - /// Note that the parameter is only used in transform of CountVectorizerModel and does not - /// affect fitting. - /// - /// Minimum term frequency - public double GetMinTF() => (double)_jvmObject.Invoke("getMinTF"); - - /// - /// Filter to ignore rare words in a document. For each document, terms with - /// frequency/count less than the given threshold are ignored. If this is an integer - /// greater than or equal to 1, then this specifies a count (of times the term must appear - /// in the document); if this is a double in [0,1), then this specifies a fraction (out of - /// the document's token count). - /// - /// Note that the parameter is only used in transform of CountVectorizerModel and does not - /// affect fitting. - /// - /// Minimum term frequency - /// - /// New with the min term frequency set - /// - public CountVectorizerModel SetMinTF(double value) => - WrapAsCountVectorizerModel((JvmObjectReference)_jvmObject.Invoke("setMinTF", value)); - - /// - /// Gets the max size of the vocabulary. will build a - /// vocabulary that only considers the top vocabSize terms ordered by term frequency across - /// the corpus. - /// - /// The max size of the vocabulary - public int GetVocabSize() => (int)_jvmObject.Invoke("getVocabSize"); - - private static CountVectorizerModel WrapAsCountVectorizerModel(object obj) => - new CountVectorizerModel((JvmObjectReference)obj); - } -} diff --git a/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs b/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs index 326268a5e..fcc90b43d 100644 --- a/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs +++ b/src/csharp/Microsoft.Spark/ML/Feature/FeatureBase.cs @@ -105,8 +105,8 @@ private static T WrapAsType(JvmObjectReference reference) .Single(c => { ParameterInfo[] parameters = c.GetParameters(); - return (parameters.Length == 1) && - (parameters[0].ParameterType == typeof(JvmObjectReference)); + return (parameters.Length == 1) && + (parameters[0].ParameterType == typeof(JvmObjectReference)); }); return (T)constructor.Invoke(new object[] {reference}); From 3c2c936b007d7b5d761fda737625dc8f7d03728b Mon Sep 17 00:00:00 2001 From: Niharika Dutta Date: Fri, 14 Aug 2020 13:32:54 -0700 Subject: [PATCH 15/20] fixing merge errors --- .gitignore | 3 +++ .../Processor/BroadcastVariableProcessor.cs | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 251cfa7e2..8e67b5699 100644 --- a/.gitignore +++ b/.gitignore @@ -367,3 +367,6 @@ hs_err_pid* # The target folder contains the output of building **/target/** + +# F# vs code +.ionide/ \ No newline at end of file diff --git a/src/csharp/Microsoft.Spark.Worker/Processor/BroadcastVariableProcessor.cs b/src/csharp/Microsoft.Spark.Worker/Processor/BroadcastVariableProcessor.cs index 41c817d02..bf8f48ed8 100644 --- a/src/csharp/Microsoft.Spark.Worker/Processor/BroadcastVariableProcessor.cs +++ b/src/csharp/Microsoft.Spark.Worker/Processor/BroadcastVariableProcessor.cs @@ -54,7 +54,8 @@ internal BroadcastVariables Process(Stream stream) else { string path = SerDe.ReadString(stream); - using FileStream fStream = File.Open(path, FileMode.Open, FileAccess.Read); + using FileStream fStream = + File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read); object value = formatter.Deserialize(fStream); BroadcastRegistry.Add(bid, value); } From 88e834d53b7be8931147a095a7b0df3c08cd9aa8 Mon Sep 17 00:00:00 2001 From: Niharika Dutta Date: Wed, 19 Aug 2020 19:24:14 -0700 Subject: [PATCH 16/20] removing ionid --- .gitignore | 2 +- .ionide/symbolCache.db | Bin 28672 -> 0 bytes 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 .ionide/symbolCache.db diff --git a/.gitignore b/.gitignore index 8e67b5699..faada9c8a 100644 --- a/.gitignore +++ b/.gitignore @@ -369,4 +369,4 @@ hs_err_pid* **/target/** # F# vs code -.ionide/ \ No newline at end of file +.ionide/ diff --git a/.ionide/symbolCache.db b/.ionide/symbolCache.db deleted file mode 100644 index 43e567d6d682d85dd32b3baebb0fdf61f67c1643..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28672 zcmeHPYiuJ|6}A(f3n*{Ao>?UI&RXX3biC7$u_ zru*Uw_|aboAq4!aPz(G7Dp3^)2^9$msX|362qA<70{*l}EvWneB<{TIImtK)+7qdu zu{?L~_%Yvi&pr3fz2}^JGgp@L0vB1UR7NW^3^wa~*(5A|iH8H;*B z&*Jr7uND(C74gzvnY~{#(YNt3Bw$FukbofpLjr~b3<($#FeG3|z>t6;0j&hSxNf$G zdV0*S4h!s^BA3}J-Ki9L<b^F{6=TjCuvK9>U*;l97n^=RUn$l~UNgYJCMRsfNA1?9Bl`LCS%K#Ngzf z;{QO@KD+;){!jcL`9JXd#`A*vLwDKru4~cx6X&wy2aZ|$+xDBbpW9ZfKeZmWylYu) z{ax!q%P(3^n}21#VfwY{Wv%+^=L?ZG(qpDCr$c=8^*P6`^IVl5<5tIVd0~tSzgigM z?z5uk`LPT6Y_-By)&wRae!(ne*4gR?lUBdKT&?7)Y>8Rp+k4w%>fhA!qkaWU!g8j& z9avyP?TH38h17hd$}v3E&T>vpABO?_g&-RIX#2Phe6h%7MQ!I9p4+7FTpy5icQ=}> z549iWIkuWzms8^H1xPCcSS58?UH(Q%WgSo}pSi&1%ghFqw{V?jb6g|G_h<$0h{v%C z-gd4n!}2^=x>Jxh}w>8SGqHC9v|D(^V*GlIoc1w|iaru~I2##}Q_|T1nBeK3-|27DujYN}U*Qnog;yWG zZr^}f59*n?f&gs=trhybzAhapG&7CzWFvl0k6CDOn7FsU92`wI{g3@Pu)FM&(n0byiegJk8tp$;dZ(^ zv=Y$fNsAXqBh!KP@Nsv!7c*PDz?GP*+?q1NVCFOrK}H;XiU+ZH0EsJTZI6;T-JAq- zjuS+DU_#0}R28`M+eFlgwc9Ox&3NKZQZlZ_NMqF!>tue`EenJqeO= z-(k%EWxt9n4P*W<`j}){81sMGVPVYwoyPqCY2B>G{GWCz|I_n-YoKm$sAt63{IKZ@ zCjVW()Axq2+xxcnw&!C{$^DLd+4ZLDu=DedKRby1-}XDUcWqa#zqMYm{LFH`_1mrC zmLIi@o8L1(L;w7AFA|##XrEvtfaXaX7#xVzi>ihYNYj*Mww$X`*YV|QzC^=M?s7b{ zR2E&Ad_Jr7vJDN1UN$i$u~P>{b1*gdEEdg`lZyRFw(CoE4YZL{_BddI=VPNxf)tFtiS1c{kwLRP_Bs2KJFK{M=Zfgp z5MzaS^-RLY01>>A4J9(PJCPk;3|-Gg3h=}8Y*2oI=KR!=4YAJvd^~4-*c;Zwp=d)e zwB3Zp8E>CHASQ8d{J&ySn^K6#J;CrWR!`-^; zqM`|6+mM`(61?aElrp4!0${VlSjO{EDzvau3op=cAg;PpUaK$*T(-!H0bn9Ea6!6~ zfK+Z2jZ}ANN{^JVURgcM@|@U<%-5<_8pe2m6F=O3P0ZtfS{ltsMe8cM8#S4aNHRO7 zP>{8>qXSDzJ6)YVwwmL`gM=7R(3d8$>Y$^w!>jYuqOlTbI-J)^}tn~k4R`# z%gp&{VwO;uNmcVHV%BjK=nOZ4Rh#YB_F$tnC4XE!;#3Ygq;Hbn}0)EjsT`7(7Hm(^n4Shvcww9bHjiGUTkb|JUFBEjjap;AiR+{ zRfi)Sw-Q%wk3G;24hEwfM_e&LA1|CP7zp+@8d-fr1qb>{%Ti8k6mY>C>QgR<x&4FUUSXg<^HJJmg z+yu~gPaO(^pg4n>Feuf`;H>7b0t(b?!3eIh&iXMNlE!-ryH6Vpc*fwWGc*$D;%gTqk&q-Cl6 z&AhPQ$ki?Yc)V6Ocw}&wR=2ebVDZ#{FgVz|M=$;yfW}jQD)4qec<>o;`)ASWbD(2j zKwGjUe(ny9$7ZuaBe${y!12mP!8_>_%6uN&A8%dEKw#jIev4{6q2zTjkIn%KC!$yb zu1H5@5XzU1b*Mvx8ey0CdUjSW00%Hm3Vam6}SWN^JQ4~XCiyMMBOimJ2yEVw`3RFM``6MP+8rh)j(l45I9&m?x{pdOk1GyTW(8~A=7nOb-8>4TyfiALgBG{yP< zfN9y}f7L(ad(C&&`$zA~ooi^`*Tu02K?}xl$2k0 zbAGCma`TY_FI()QQ7l(SGLY<5OiOdr+o8!4+e1*~+h_)dnql67B-dwfHZZUiz zzabvn1HdDMHf^Ll5)zb@WM2U=c;2LBe==J4CfT-n1+TU#`8q*c)H!K{K&_voHgaIsS6#Dmy)4GV_@uWLz6 zCQj|6Ygy;mRq^1efCpaJLU|n1bS;|z2Cr*L98HWhx|R)qg4eYq#uF1dLT%c$kd6vq zg6CvS^dv?aO!Sley-p$ur@`DqgVL}*9Lx0e7KVs&&S^rGFGR!5--h~O1_78cOIj<(HBye+O7eATP3F@vFj0ZfVApl|kbfCFCq0p*~C8rNO| z5O|G}`oPsFL8V8~HcD~+AGF&{{x#oUeXn>w_CEI7Jzw?M-QRS_UEg(`cE0b-IsWD- z*gvut>CcRpApt`Ih6D@=7!vqDl0cTI`}N6#m<}|fmUX}y7k4Wa4`L?jmVGkJ6*QPm zYp1g@6xEwkmnZ@8xJSUAea_QpR|;ZeGEFy$x=)|HN*vD)pBhe0C^NlncK8B;yY%7BE>k_JC|$KEu;nbLo^u3l z$A=f=Oa&b*Noz^uG)L4O_hgO6Lz)T9gtSOo1n!-O7mYD7>XTNG)BbMWD7*yKTnn{p2kcErn^;5QO|^~@dfjb=g`(vO!rQ!^N%?io_SZ? zsF<0ra;!kxqpY{^imti@RPfs8^z`%@jfgc_8rKLgN6`#2wD@X-UA_4A%qCGc(UU6~ zuc#A&3DvB(O|fV<3^&xn+pKdkFx9uS8&=()>Y`TKtVBwi1a7T|SN5q>Q|Fbs-R8Wi r5i_V}JR8*1*wjg2b^;fqlb8T7-kvp6;i)FU08O@N<|Vu8nsWLNN`{i9 From ebfd45a633ab20bd192516e606d8febaaddf2514 Mon Sep 17 00:00:00 2001 From: Niharika Dutta Date: Wed, 4 Nov 2020 11:56:15 -0800 Subject: [PATCH 17/20] Fix for calling Broadcast in Databricks --- src/csharp/Microsoft.Spark/Broadcast.cs | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/csharp/Microsoft.Spark/Broadcast.cs b/src/csharp/Microsoft.Spark/Broadcast.cs index f0ea061fb..0a21bfa00 100644 --- a/src/csharp/Microsoft.Spark/Broadcast.cs +++ b/src/csharp/Microsoft.Spark/Broadcast.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using static System.Environment; using System.IO; using System.Net; using System.Runtime.Serialization; @@ -171,11 +172,26 @@ private JvmObjectReference CreateBroadcast_V2_3_2_AndAbove( // Spark versions. bool encryptionEnabled = bool.Parse( sc.GetConf().Get("spark.io.encryption.enabled", "false")); + JvmObjectReference _pythonBroadcast; + bool isDatabricks = + !string.IsNullOrEmpty(GetEnvironmentVariable("DATABRICKS_RUNTIME_VERSION")); - var _pythonBroadcast = (JvmObjectReference)javaSparkContext.Jvm.CallStaticJavaMethod( - "org.apache.spark.api.python.PythonRDD", - "setupBroadcast", - _path); + // Databricks has changed the signature of setupBroadcast in its Spark + if (isDatabricks) + { + _pythonBroadcast = (JvmObjectReference)javaSparkContext.Jvm.CallStaticJavaMethod( + "org.apache.spark.api.python.PythonRDD", + "setupBroadcast", + javaSparkContext, + _path); + } + else + { + _pythonBroadcast = (JvmObjectReference)javaSparkContext.Jvm.CallStaticJavaMethod( + "org.apache.spark.api.python.PythonRDD", + "setupBroadcast", + _path); + } if (encryptionEnabled) { From c01b36f6b7e9566cfcbd2ca69f26dd0702612373 Mon Sep 17 00:00:00 2001 From: Niharika Dutta Date: Fri, 6 Nov 2020 11:38:25 -0800 Subject: [PATCH 18/20] moving IsDatabricks util to ConfigurationService --- src/csharp/Microsoft.Spark.Worker/DaemonWorker.cs | 2 +- .../Microsoft.Spark.Worker/Processor/PayloadProcessor.cs | 3 ++- src/csharp/Microsoft.Spark.Worker/Utils/SettingUtils.cs | 3 --- src/csharp/Microsoft.Spark/Broadcast.cs | 4 +--- src/csharp/Microsoft.Spark/Services/ConfigurationService.cs | 3 +++ 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/csharp/Microsoft.Spark.Worker/DaemonWorker.cs b/src/csharp/Microsoft.Spark.Worker/DaemonWorker.cs index c4ef752b9..ecbd3c272 100644 --- a/src/csharp/Microsoft.Spark.Worker/DaemonWorker.cs +++ b/src/csharp/Microsoft.Spark.Worker/DaemonWorker.cs @@ -128,7 +128,7 @@ private void StartServer(ISocketWrapper listener) SerDe.Write(socket.OutputStream, taskRunnerId); socket.OutputStream.Flush(); - if (Utils.SettingUtils.IsDatabricks) + if (ConfigurationService.IsDatabricks) { SerDe.ReadString(socket.InputStream); } diff --git a/src/csharp/Microsoft.Spark.Worker/Processor/PayloadProcessor.cs b/src/csharp/Microsoft.Spark.Worker/Processor/PayloadProcessor.cs index 58dd588aa..0f33d582e 100644 --- a/src/csharp/Microsoft.Spark.Worker/Processor/PayloadProcessor.cs +++ b/src/csharp/Microsoft.Spark.Worker/Processor/PayloadProcessor.cs @@ -7,6 +7,7 @@ using System.Collections.Generic; using System.IO; using Microsoft.Spark.Interop.Ipc; +using Microsoft.Spark.Services; using Microsoft.Spark.Worker.Utils; namespace Microsoft.Spark.Worker.Processor @@ -68,7 +69,7 @@ internal Payload Process(Stream stream) // our current context. AssemblyLoaderHelper.RegisterAssemblyHandler(); - if (SettingUtils.IsDatabricks) + if (ConfigurationService.IsDatabricks) { SerDe.ReadString(stream); SerDe.ReadString(stream); diff --git a/src/csharp/Microsoft.Spark.Worker/Utils/SettingUtils.cs b/src/csharp/Microsoft.Spark.Worker/Utils/SettingUtils.cs index 50262a38c..c48f1618a 100644 --- a/src/csharp/Microsoft.Spark.Worker/Utils/SettingUtils.cs +++ b/src/csharp/Microsoft.Spark.Worker/Utils/SettingUtils.cs @@ -12,9 +12,6 @@ namespace Microsoft.Spark.Worker.Utils /// internal static class SettingUtils { - internal static bool IsDatabricks { get; } = - !string.IsNullOrEmpty(GetEnvironmentVariable("DATABRICKS_RUNTIME_VERSION")); - internal static string GetWorkerFactorySecret(Version version) { return (version >= new Version(Versions.V2_3_1)) ? diff --git a/src/csharp/Microsoft.Spark/Broadcast.cs b/src/csharp/Microsoft.Spark/Broadcast.cs index 0a21bfa00..34d02b1da 100644 --- a/src/csharp/Microsoft.Spark/Broadcast.cs +++ b/src/csharp/Microsoft.Spark/Broadcast.cs @@ -173,11 +173,9 @@ private JvmObjectReference CreateBroadcast_V2_3_2_AndAbove( bool encryptionEnabled = bool.Parse( sc.GetConf().Get("spark.io.encryption.enabled", "false")); JvmObjectReference _pythonBroadcast; - bool isDatabricks = - !string.IsNullOrEmpty(GetEnvironmentVariable("DATABRICKS_RUNTIME_VERSION")); // Databricks has changed the signature of setupBroadcast in its Spark - if (isDatabricks) + if (ConfigurationService.IsDatabricks) { _pythonBroadcast = (JvmObjectReference)javaSparkContext.Jvm.CallStaticJavaMethod( "org.apache.spark.api.python.PythonRDD", diff --git a/src/csharp/Microsoft.Spark/Services/ConfigurationService.cs b/src/csharp/Microsoft.Spark/Services/ConfigurationService.cs index 53e9d9126..82b4cee76 100644 --- a/src/csharp/Microsoft.Spark/Services/ConfigurationService.cs +++ b/src/csharp/Microsoft.Spark/Services/ConfigurationService.cs @@ -5,6 +5,7 @@ using System; using System.IO; using System.Runtime.InteropServices; +using static System.Environment; using Microsoft.Spark.Utils; namespace Microsoft.Spark.Services @@ -15,6 +16,8 @@ namespace Microsoft.Spark.Services /// internal sealed class ConfigurationService : IConfigurationService { + internal static bool IsDatabricks { get; } = + !string.IsNullOrEmpty(GetEnvironmentVariable("DATABRICKS_RUNTIME_VERSION")); public const string WorkerDirEnvVarName = "DOTNET_WORKER_DIR"; public const string WorkerReadBufferSizeEnvVarName = "spark.dotnet.worker.readBufferSize"; public const string WorkerWriteBufferSizeEnvVarName = From 105478b2f162bcb59bfe3d021fa12a878b16407e Mon Sep 17 00:00:00 2001 From: Niharika Dutta Date: Fri, 6 Nov 2020 14:21:42 -0800 Subject: [PATCH 19/20] PR comments --- src/csharp/Microsoft.Spark/Broadcast.cs | 1 - src/csharp/Microsoft.Spark/Services/ConfigurationService.cs | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/csharp/Microsoft.Spark/Broadcast.cs b/src/csharp/Microsoft.Spark/Broadcast.cs index 34d02b1da..016ec90b6 100644 --- a/src/csharp/Microsoft.Spark/Broadcast.cs +++ b/src/csharp/Microsoft.Spark/Broadcast.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using static System.Environment; using System.IO; using System.Net; using System.Runtime.Serialization; diff --git a/src/csharp/Microsoft.Spark/Services/ConfigurationService.cs b/src/csharp/Microsoft.Spark/Services/ConfigurationService.cs index 82b4cee76..74098de59 100644 --- a/src/csharp/Microsoft.Spark/Services/ConfigurationService.cs +++ b/src/csharp/Microsoft.Spark/Services/ConfigurationService.cs @@ -16,8 +16,6 @@ namespace Microsoft.Spark.Services /// internal sealed class ConfigurationService : IConfigurationService { - internal static bool IsDatabricks { get; } = - !string.IsNullOrEmpty(GetEnvironmentVariable("DATABRICKS_RUNTIME_VERSION")); public const string WorkerDirEnvVarName = "DOTNET_WORKER_DIR"; public const string WorkerReadBufferSizeEnvVarName = "spark.dotnet.worker.readBufferSize"; public const string WorkerWriteBufferSizeEnvVarName = @@ -52,6 +50,9 @@ public TimeSpan JvmThreadGCInterval } } + internal static bool IsDatabricks { get; } = + !string.IsNullOrEmpty(GetEnvironmentVariable("DATABRICKS_RUNTIME_VERSION")); + /// /// Returns the port number for socket communication between JVM and CLR. /// From d414755a12e69ea07198f0929ad5128423986699 Mon Sep 17 00:00:00 2001 From: Niharika Dutta Date: Wed, 11 Nov 2020 16:26:07 -0800 Subject: [PATCH 20/20] Update src/csharp/Microsoft.Spark/Broadcast.cs Co-authored-by: Terry Kim --- src/csharp/Microsoft.Spark/Broadcast.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Microsoft.Spark/Broadcast.cs b/src/csharp/Microsoft.Spark/Broadcast.cs index 016ec90b6..8f833dbb0 100644 --- a/src/csharp/Microsoft.Spark/Broadcast.cs +++ b/src/csharp/Microsoft.Spark/Broadcast.cs @@ -173,7 +173,7 @@ private JvmObjectReference CreateBroadcast_V2_3_2_AndAbove( sc.GetConf().Get("spark.io.encryption.enabled", "false")); JvmObjectReference _pythonBroadcast; - // Databricks has changed the signature of setupBroadcast in its Spark + // Spark in Databricks is different from OSS Spark and requires to pass the SparkContext object to setupBroadcast. if (ConfigurationService.IsDatabricks) { _pythonBroadcast = (JvmObjectReference)javaSparkContext.Jvm.CallStaticJavaMethod(