.Net MEVD: Test coverage for all kinds of data fields supported by Azure AI Search

dotnet/src/Connectors/Connectors.AzureAISearch.UnitTests/AzureAISearchCollectionCreateMappingTests.cs

@@ -214,7 +214,6 @@ public static IEnumerable<object[]> DataTypeMappingOptions()
         yield return new object[] { typeof(long), SearchFieldDataType.Int64 };
         yield return new object[] { typeof(float), SearchFieldDataType.Double };
         yield return new object[] { typeof(double), SearchFieldDataType.Double };
-        yield return new object[] { typeof(DateTime), SearchFieldDataType.DateTimeOffset };
         yield return new object[] { typeof(DateTimeOffset), SearchFieldDataType.DateTimeOffset };
 
         yield return new object[] { typeof(string[]), SearchFieldDataType.Collection(SearchFieldDataType.String) };

dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchCollectionCreateMapping.cs

@@ -121,9 +121,10 @@ public static SearchFieldDataType GetSDKFieldDataType(Type propertyType)
             Type t when t == typeof(bool) => SearchFieldDataType.Boolean,
             Type t when t == typeof(int) => SearchFieldDataType.Int32,
             Type t when t == typeof(long) => SearchFieldDataType.Int64,
+            // We don't map float to SearchFieldDataType.Single, because Azure AI Search doesn't support it.
+            // Half is also listed by the SDK, but currently not supported.
             Type t when t == typeof(float) => SearchFieldDataType.Double,
             Type t when t == typeof(double) => SearchFieldDataType.Double,
-            Type t when t == typeof(DateTime) => SearchFieldDataType.DateTimeOffset,
             Type t when t == typeof(DateTimeOffset) => SearchFieldDataType.DateTimeOffset,
 
             Type t when t == typeof(string[]) => SearchFieldDataType.Collection(SearchFieldDataType.String),
@@ -138,8 +139,6 @@ public static SearchFieldDataType GetSDKFieldDataType(Type propertyType)
             Type t when t == typeof(List<float>) => SearchFieldDataType.Collection(SearchFieldDataType.Double),
             Type t when t == typeof(double[]) => SearchFieldDataType.Collection(SearchFieldDataType.Double),
             Type t when t == typeof(List<double>) => SearchFieldDataType.Collection(SearchFieldDataType.Double),
-            Type t when t == typeof(DateTime[]) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
-            Type t when t == typeof(List<DateTime>) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
             Type t when t == typeof(DateTimeOffset[]) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
             Type t when t == typeof(List<DateTimeOffset>) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
 

dotnet/src/VectorDataIntegrationTests/AzureAISearchIntegrationTests/CRUD/AzureAISearchAllSupportedTypesTests.cs

@@ -0,0 +1,106 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using AzureAISearchIntegrationTests.Support;
+using Microsoft.Extensions.VectorData;
+using VectorDataSpecificationTests.Xunit;
+using Xunit;
+
+namespace AzureAISearchIntegrationTests.CRUD;
+
+public class AzureAISearchAllSupportedTypesTests(AzureAISearchFixture fixture) : IClassFixture<AzureAISearchFixture>
+{
+    [ConditionalFact]
+    public async Task AllTypesBatchGetAsync()
+    {
+        var collection = fixture.TestStore.DefaultVectorStore.GetCollection<string, AzureAISearchAllTypes>("all-types", AzureAISearchAllTypes.GetRecordDefinition());
+        await collection.EnsureCollectionExistsAsync();
+
+        List<AzureAISearchAllTypes> records =
+        [
+            new()
+            {
+                Id = "all-types-1",
+                BoolProperty = true,
+                NullableBoolProperty = false,
+                StringProperty = "string prop 1",
+                NullableStringProperty = "nullable prop 1",
+                IntProperty = 1,
+                NullableIntProperty = 10,
+                LongProperty = 100L,
+                NullableLongProperty = 1000L,
+                FloatProperty = 10.5f,
+                NullableFloatProperty = 100.5f,
+                DoubleProperty = 23.75d,
+                NullableDoubleProperty = 233.75d,
+                DateTimeOffsetProperty = DateTimeOffset.UtcNow,
+                NullableDateTimeOffsetProperty = DateTimeOffset.UtcNow,
+                StringArray = ["one", "two"],
+                StringList = ["eleven", "twelve"],
+                BoolArray = [true, false],
+                BoolList = [true, false],
+                IntArray = [1, 2],
+                IntList = [11, 12],
+                LongArray = [100L, 200L],
+                LongList = [1100L, 1200L],
+                FloatArray = [1.5f, 2.5f],
+                FloatList = [11.5f, 12.5f],
+                DoubleArray = [1.5d, 2.5d],
+                DoubleList = [11.5d, 12.5d],
+                DateTimeOffsetArray = [DateTimeOffset.UtcNow, DateTimeOffset.UtcNow],
+                DateTimeOffsetList = [DateTimeOffset.UtcNow, DateTimeOffset.UtcNow],
+                Embedding = new ReadOnlyMemory<float>([1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f, 8.5f])
+            },
+            new()
+            {
+                Id = "all-types-2",
+                BoolProperty = false,
+                NullableBoolProperty = null,
+                StringProperty = "string prop 2",
+                NullableStringProperty = null,
+                IntProperty = 2,
+                NullableIntProperty = null,
+                LongProperty = 200L,
+                NullableLongProperty = null,
+                FloatProperty = 20.5f,
+                NullableFloatProperty = null,
+                DoubleProperty = 43.75,
+                NullableDoubleProperty = null,
+                Embedding = ReadOnlyMemory<float>.Empty,
+                // From https://learn.microsoft.com/en-us/rest/api/searchservice/supported-data-types:
+                // "All of the above types are nullable, except for collections of primitive and complex types, for example, Collection(Edm.String)"
+                // So for collections, we can't use nulls.
+                StringArray = [],
+                StringList = [],
+                BoolArray = [],
+                BoolList = [],
+                IntArray = [],
+                IntList = [],
+                LongArray = [],
+                LongList = [],
+                FloatArray = [],
+                FloatList = [],
+                DoubleArray = [],
+                DoubleList = [],
+                DateTimeOffsetArray = [],
+                DateTimeOffsetList = [],
+            }
+        ];
+
+        try
+        {
+            await collection.UpsertAsync(records);
+
+            var allTypes = await collection.GetAsync(records.Select(r => r.Id), new RecordRetrievalOptions { IncludeVectors = true }).ToListAsync();
+
+            var allTypes1 = allTypes.Single(x => x.Id == records[0].Id);
+            var allTypes2 = allTypes.Single(x => x.Id == records[1].Id);
+
+            records[0].AssertEqual(allTypes1);
+            records[1].AssertEqual(allTypes2);
+        }
+        finally
+        {
+            await collection.EnsureCollectionDeletedAsync();
+        }
+    }
+}

dotnet/src/VectorDataIntegrationTests/AzureAISearchIntegrationTests/Support/AzureAISearchAllTypes.cs

@@ -0,0 +1,169 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.VectorData;
+using Xunit;
+
+namespace AzureAISearchIntegrationTests.Support;
+
+#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
+#pragma warning disable CA1819 // Properties should not return arrays
+
+public class AzureAISearchAllTypes
+{
+    [VectorStoreKey]
+    public string Id { get; set; }
+
+    [VectorStoreData]
+    public bool BoolProperty { get; set; }
+    [VectorStoreData]
+    public bool? NullableBoolProperty { get; set; }
+    [VectorStoreData]
+    public string StringProperty { get; set; }
+    [VectorStoreData]
+    public string? NullableStringProperty { get; set; }
+    [VectorStoreData]
+    public int IntProperty { get; set; }
+    [VectorStoreData]
+    public int? NullableIntProperty { get; set; }
+    [VectorStoreData]
+    public long LongProperty { get; set; }
+    [VectorStoreData]
+    public long? NullableLongProperty { get; set; }
+    [VectorStoreData]
+    public float FloatProperty { get; set; }
+    [VectorStoreData]
+    public float? NullableFloatProperty { get; set; }
+    [VectorStoreData]
+    public double DoubleProperty { get; set; }
+    [VectorStoreData]
+    public double? NullableDoubleProperty { get; set; }
+    [VectorStoreData]
+    public DateTimeOffset DateTimeOffsetProperty { get; set; }
+    [VectorStoreData]
+    public DateTimeOffset? NullableDateTimeOffsetProperty { get; set; }
+
+    [VectorStoreData]
+    public string[] StringArray { get; set; }
+    [VectorStoreData]
+    public List<string> StringList { get; set; }
+    [VectorStoreData]
+    public bool[] BoolArray { get; set; }
+    [VectorStoreData]
+    public List<bool> BoolList { get; set; }
+    [VectorStoreData]
+    public int[] IntArray { get; set; }
+    [VectorStoreData]
+    public List<int> IntList { get; set; }
+    [VectorStoreData]
+    public long[] LongArray { get; set; }
+    [VectorStoreData]
+    public List<long> LongList { get; set; }
+    [VectorStoreData]
+    public float[] FloatArray { get; set; }
+    [VectorStoreData]
+    public List<float> FloatList { get; set; }
+    [VectorStoreData]
+    public double[] DoubleArray { get; set; }
+    [VectorStoreData]
+    public List<double> DoubleList { get; set; }
+    [VectorStoreData]
+    public DateTimeOffset[] DateTimeOffsetArray { get; set; }
+    [VectorStoreData]
+    public List<DateTimeOffset> DateTimeOffsetList { get; set; }
+
+    [VectorStoreVector(Dimensions: 8, DistanceFunction = DistanceFunction.DotProductSimilarity)]
+    public ReadOnlyMemory<float>? Embedding { get; set; }
+
+    internal void AssertEqual(AzureAISearchAllTypes other)
+    {
+        Assert.Equal(this.Id, other.Id);
+        Assert.Equal(this.BoolProperty, other.BoolProperty);
+        Assert.Equal(this.NullableBoolProperty, other.NullableBoolProperty);
+        Assert.Equal(this.StringProperty, other.StringProperty);
+        Assert.Equal(this.NullableStringProperty, other.NullableStringProperty);
+        Assert.Equal(this.IntProperty, other.IntProperty);
+        Assert.Equal(this.NullableIntProperty, other.NullableIntProperty);
+        Assert.Equal(this.LongProperty, other.LongProperty);
+        Assert.Equal(this.NullableLongProperty, other.NullableLongProperty);
+        Assert.Equal(this.FloatProperty, other.FloatProperty);
+        Assert.Equal(this.NullableFloatProperty, other.NullableFloatProperty);
+        Assert.Equal(this.DoubleProperty, other.DoubleProperty);
+        Assert.Equal(this.NullableDoubleProperty, other.NullableDoubleProperty);
+        AssertEqual(this.DateTimeOffsetProperty, other.DateTimeOffsetProperty);
+        Assert.Equal(this.NullableDateTimeOffsetProperty.HasValue, other.NullableDateTimeOffsetProperty.HasValue);
+        if (this.NullableDateTimeOffsetProperty.HasValue && other.NullableDateTimeOffsetProperty.HasValue)
+        {
+            AssertEqual(this.NullableDateTimeOffsetProperty.Value, other.NullableDateTimeOffsetProperty.Value);
+        }
+
+        Assert.Equal(this.StringArray, other.StringArray);
+        Assert.Equal(this.StringList, other.StringList);
+        Assert.Equal(this.BoolArray, other.BoolArray);
+        Assert.Equal(this.BoolList, other.BoolList);
+        Assert.Equal(this.IntArray, other.IntArray);
+        Assert.Equal(this.IntList, other.IntList);
+        Assert.Equal(this.LongArray, other.LongArray);
+        Assert.Equal(this.LongList, other.LongList);
+        Assert.Equal(this.FloatArray, other.FloatArray);
+        Assert.Equal(this.FloatList, other.FloatList);
+        Assert.Equal(this.DoubleArray, other.DoubleArray);
+        Assert.Equal(this.DoubleList, other.DoubleList);
+        Assert.Equal(this.DateTimeOffsetArray.Length, other.DateTimeOffsetArray.Length);
+        for (int i = 0; i < this.DateTimeOffsetArray.Length; i++)
+        {
+            AssertEqual(this.DateTimeOffsetArray[i], other.DateTimeOffsetArray[i]);
+        }
+        Assert.Equal(this.DateTimeOffsetList.Count, other.DateTimeOffsetList.Count);
+        for (int i = 0; i < this.DateTimeOffsetList.Count; i++)
+        {
+            AssertEqual(this.DateTimeOffsetList[i], other.DateTimeOffsetList[i]);
+        }
+
+        Assert.Equal(this.Embedding!.Value.ToArray(), other.Embedding!.Value.ToArray());
+
+        static void AssertEqual(DateTimeOffset expected, DateTimeOffset actual)
+        {
+            Assert.Equal(expected, actual, TimeSpan.FromSeconds(0.01));
+        }
+    }
+
+    internal static VectorStoreRecordDefinition GetRecordDefinition()
+        => new()
+        {
+            Properties =
+            [
+                new VectorStoreKeyProperty(nameof(AzureAISearchAllTypes.Id), typeof(string)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.BoolProperty), typeof(bool)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.NullableBoolProperty), typeof(bool?)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.StringProperty), typeof(string)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.NullableStringProperty), typeof(string)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.IntProperty), typeof(int)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.NullableIntProperty), typeof(int?)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.LongProperty), typeof(long)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.NullableLongProperty), typeof(long?)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.FloatProperty), typeof(float)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.NullableFloatProperty), typeof(float?)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.DoubleProperty), typeof(double)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.NullableDoubleProperty), typeof(double?)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.DateTimeOffsetProperty), typeof(DateTimeOffset)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.NullableDateTimeOffsetProperty), typeof(DateTimeOffset?)),
+
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.StringArray), typeof(string[])),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.StringList), typeof(List<string>)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.BoolArray), typeof(bool[])),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.BoolList), typeof(List<bool>)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.IntArray), typeof(int[])),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.IntList), typeof(List<int>)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.LongArray), typeof(long[])),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.LongList), typeof(List<long>)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.FloatArray), typeof(float[])),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.FloatList), typeof(List<float>)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.DoubleArray), typeof(double[])),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.DoubleList), typeof(List<double>)),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.DateTimeOffsetArray), typeof(DateTimeOffset[])),
+                new VectorStoreDataProperty(nameof(AzureAISearchAllTypes.DateTimeOffsetList), typeof(List<DateTimeOffset>)),
+
+                new VectorStoreVectorProperty(nameof(AzureAISearchAllTypes.Embedding), typeof(ReadOnlyMemory<float>?), 8) { DistanceFunction = Microsoft.Extensions.VectorData.DistanceFunction.DotProductSimilarity }
+            ]
+        };
+}