Fix $filter on searchable string fields (#27)

AzureSearchEmulator.IntegrationTests/EmulatorIntegrationTests.cs

@@ -149,6 +149,90 @@ public async Task SearchDocuments_WithFilter_ShouldReturnResults()
         await indexClient.DeleteIndexAsync(indexName);
     }
 
+    [Fact]
+    public async Task SearchDocuments_WithFilter_OnSearchableStringField_ShouldReturnResults()
+    {
+        // Regression test for issue #27: filtering a Searchable+Filterable string field
+        // must work. Before the fix, the analyzer lowercased 'Electronics' at index time
+        // and the exact TermQuery at filter time found zero hits.
+        const string indexName = "test-filter-searchable-string";
+        var indexClient = factory.CreateSearchIndexClient();
+        var searchClient = factory.CreateSearchClient(indexName);
+
+        await CreateIndexAsync(indexClient, indexName);
+        await UploadDocumentsAsync(searchClient);
+
+        var options = new SearchOptions
+        {
+            Filter = "Category eq 'Electronics'",
+            Size = 50
+        };
+        var results = await searchClient.SearchAsync<Product>("*", options);
+
+        Assert.NotNull(results);
+        var items = await results.Value.GetResultsAsync().ToListAsync();
+        Assert.NotEmpty(items);
+        Assert.True(items.All(r => r.Document.Category == "Electronics"),
+            "All results should have Category = Electronics");
+
+        await indexClient.DeleteIndexAsync(indexName);
+    }
+
+    [Fact]
+    public async Task SearchDocuments_SearchOnDualIndexedField_StillMatchesAnalyzedTokens()
+    {
+        // Dual-indexing (TextField + StringField under same name) must not break
+        // full-text search: a tokenized search for 'electronics' must still match.
+        const string indexName = "test-search-on-dual-indexed";
+        var indexClient = factory.CreateSearchIndexClient();
+        var searchClient = factory.CreateSearchClient(indexName);
+
+        await CreateIndexAsync(indexClient, indexName);
+        await UploadDocumentsAsync(searchClient);
+
+        var options = new SearchOptions { SearchFields = { "Category" }, Size = 50 };
+        var results = await searchClient.SearchAsync<Product>("electronics", options);
+        var items = await results.Value.GetResultsAsync().ToListAsync();
+
+        Assert.NotEmpty(items);
+        Assert.True(items.All(r => r.Document.Category == "Electronics"));
+
+        await indexClient.DeleteIndexAsync(indexName);
+    }
+
+    [Fact]
+    public async Task SearchDocuments_FilterOnNonFilterableField_ShouldError()
+    {
+        // Azure rejects $filter against a non-filterable field; the emulator should too.
+        const string indexName = "test-filter-nonfilterable";
+        var indexClient = factory.CreateSearchIndexClient();
+        var searchClient = factory.CreateSearchClient(indexName);
+
+        var index = new SearchIndex(indexName)
+        {
+            Fields =
+            [
+                new SearchField(nameof(Product.Id), SearchFieldDataType.String) { IsKey = true, IsStored = true, IsSearchable = true, IsFilterable = true },
+                new SearchField(nameof(Product.Name), SearchFieldDataType.String) { IsSearchable = true, IsFilterable = false, IsStored = true },
+                new SearchField(nameof(Product.Description), SearchFieldDataType.String) { IsSearchable = true, IsStored = true },
+                new SearchField(nameof(Product.Price), SearchFieldDataType.Double) { IsFilterable = true, IsSortable = true, IsStored = true },
+                new SearchField(nameof(Product.Category), SearchFieldDataType.String) { IsSearchable = true, IsFilterable = true, IsStored = true },
+                new SearchField(nameof(Product.InStock), SearchFieldDataType.Boolean) { IsFilterable = true, IsStored = true }
+            ]
+        };
+        await indexClient.CreateIndexAsync(index);
+        await UploadDocumentsAsync(searchClient);
+
+        var options = new SearchOptions { Filter = "Name eq 'Laptop Pro 15'", Size = 50 };
+        await Assert.ThrowsAnyAsync<Exception>(async () =>
+        {
+            var results = await searchClient.SearchAsync<Product>("*", options);
+            await results.Value.GetResultsAsync().ToListAsync();
+        });
+
+        await indexClient.DeleteIndexAsync(indexName);
+    }
+
     [Fact]
     public async Task SearchDocuments_WithSorting_ShouldReturnSortedResults()
     {
@@ -464,7 +548,7 @@ private static async Task<SearchIndex> CreateIndexAsync(SearchIndexClient indexC
                 new SearchField(nameof(Product.Name), SearchFieldDataType.String) { IsSearchable = true, IsStored = true },
                 new SearchField(nameof(Product.Description), SearchFieldDataType.String) { IsSearchable = true, IsStored = true},
                 new SearchField(nameof(Product.Price), SearchFieldDataType.Double) { IsFilterable = true, IsSortable = true, IsStored = true },
-                new SearchField(nameof(Product.Category), SearchFieldDataType.String) { IsFilterable = true, IsStored = true },
+                new SearchField(nameof(Product.Category), SearchFieldDataType.String) { IsSearchable = true, IsFilterable = true, IsStored = true },
                 new SearchField(nameof(Product.InStock), SearchFieldDataType.Boolean) { IsFilterable = true, IsStored = true }
             ]
         };

AzureSearchEmulator.UnitTests/LuceneNetIndexSearcherTests.cs

@@ -1,3 +1,5 @@
+using System.Text.Json.Nodes;
+using AzureSearchEmulator.Indexing;
 using AzureSearchEmulator.Models;
 using AzureSearchEmulator.SearchData;
 using AzureSearchEmulator.Searching;
@@ -928,3 +930,132 @@ private class StubIndexReaderFactory(Lucene.Net.Store.Directory directory) : ILu
         public void ClearCachedReader(string indexName) { }
     }
 }
+
+/// <summary>
+/// Tests covering filter behavior for string fields indexed through the real
+/// SearchFieldExtensions.CreateField path (which applies the per-field analyzer),
+/// exercising combinations of Searchable and Filterable. See issue #27.
+/// </summary>
+public class LuceneNetIndexSearcher_SearchableFilterableTests
+{
+    private static (LuceneTestHelper helper, LuceneNetIndexSearcher searcher) Build(SearchIndex index, List<JsonObject> docs)
+    {
+        var luceneDocs = docs.Select(d =>
+        {
+            var doc = new Lucene.Net.Documents.Document();
+            foreach (var field in index.Fields)
+            {
+                if (d[field.Name] is { } value)
+                {
+                    foreach (var f in field.CreateFields(value))
+                    {
+                        doc.Add(f);
+                    }
+                }
+            }
+            return doc;
+        }).ToList();
+
+        var helper = new LuceneTestHelper(index, luceneDocs);
+        var searcher = new LuceneNetIndexSearcher(new StubReaderFactory(helper.Directory));
+        return (helper, searcher);
+    }
+
+    [Fact]
+    public async Task Filter_StringEquality_SearchableAndFilterable_ReturnsMatch()
+    {
+        var index = new SearchIndex
+        {
+            Name = "products",
+            Fields =
+            [
+                new SearchField { Name = "Id", Type = "Edm.String", Key = true, Searchable = false, Filterable = true },
+                new SearchField { Name = "Category", Type = "Edm.String", Searchable = true, Filterable = true },
+            ]
+        };
+        var docs = new List<JsonObject>
+        {
+            new() { ["Id"] = "1", ["Category"] = "Electronics" },
+            new() { ["Id"] = "2", ["Category"] = "Books" },
+        };
+        var (helper, searcher) = Build(index, docs);
+        using var _ = helper;
+
+        var response = await searcher.Search(index, new SearchRequest
+        {
+            Search = "*",
+            Filter = "Category eq 'Electronics'",
+            Top = 50
+        });
+
+        Assert.Single(response.Results);
+        Assert.Equal("1", response.Results[0]["Id"]?.GetValue<string>());
+    }
+
+    [Fact]
+    public async Task Filter_StringEquality_Defaults_ReturnsMatch()
+    {
+        // Mirrors the bug reported in issue #27: fields where Searchable and Filterable
+        // are unset (both default to true for Edm.String) must still filter correctly.
+        var index = new SearchIndex
+        {
+            Name = "products",
+            Fields =
+            [
+                new SearchField { Name = "Id", Type = "Edm.String", Key = true, Searchable = true, Filterable = true },
+                new SearchField { Name = "Category", Type = "Edm.String" },
+            ]
+        };
+        var docs = new List<JsonObject>
+        {
+            new() { ["Id"] = "1", ["Category"] = "Electronics" },
+            new() { ["Id"] = "2", ["Category"] = "Books" },
+        };
+        var (helper, searcher) = Build(index, docs);
+        using var _ = helper;
+
+        var response = await searcher.Search(index, new SearchRequest
+        {
+            Search = "*",
+            Filter = "Category eq 'Electronics'",
+            Top = 50
+        });
+
+        Assert.Single(response.Results);
+        Assert.Equal("1", response.Results[0]["Id"]?.GetValue<string>());
+    }
+
+    [Fact]
+    public async Task Filter_StringEquality_SearchableNotFilterable_ShouldError()
+    {
+        var index = new SearchIndex
+        {
+            Name = "posts",
+            Fields =
+            [
+                new SearchField { Name = "Id", Type = "Edm.String", Key = true, Searchable = false, Filterable = true },
+                new SearchField { Name = "Body", Type = "Edm.String", Searchable = true, Filterable = false },
+            ]
+        };
+        var docs = new List<JsonObject>
+        {
+            new() { ["Id"] = "1", ["Body"] = "Hello world" },
+        };
+        var (helper, searcher) = Build(index, docs);
+        using var _ = helper;
+
+        await Assert.ThrowsAnyAsync<Exception>(() => searcher.Search(index, new SearchRequest
+        {
+            Search = "*",
+            Filter = "Body eq 'Hello world'",
+            Top = 50
+        }));
+    }
+
+    private class StubReaderFactory(Lucene.Net.Store.Directory directory) : ILuceneIndexReaderFactory
+    {
+        public IndexReader GetIndexReader(string indexName) => DirectoryReader.Open(directory);
+        public IndexReader RefreshReader(string indexName) => GetIndexReader(indexName);
+        public void ClearCachedReader(string indexName) { }
+    }
+}

AzureSearchEmulator/Indexing/SearchFieldExtensions.cs

@@ -7,14 +7,35 @@ namespace AzureSearchEmulator.Indexing;
 
 public static class SearchFieldExtensions
 {
-    public static IIndexableField CreateField(this SearchField field, JsonNode value)
+    public static IEnumerable<IIndexableField> CreateFields(this SearchField field, JsonNode value)
     {
         var stored = field.Retrievable ? Field.Store.YES : Field.Store.NO;
 
-        return field.Type switch
+        if (field.Type == "Edm.String")
+        {
+            var str = value.GetValue<string>();
+            var searchable = field.Searchable.GetValueOrDefault(true);
+            var filterable = field.Filterable;
+
+            if (searchable)
+            {
+                yield return new TextField(field.Name, str, stored);
+                // Filter/sort/facet require a non-analyzed copy under the same field name
+                // so TermQuery-based filters match the raw literal (matches Azure semantics).
+                if (filterable || field.Sortable.GetValueOrDefault() || field.Facetable.GetValueOrDefault())
+                {
+                    yield return new StringField(field.Name, str, Field.Store.NO);
+                }
+            }
+            else
+            {
+                yield return new StringField(field.Name, str, stored);
+            }
+            yield break;
+        }
+
+        yield return field.Type switch
         {
-            "Edm.String" when field.Searchable.GetValueOrDefault(true) => new TextField(field.Name, value.GetValue<string>(), stored),
-            "Edm.String" when !field.Searchable.GetValueOrDefault(true) => new StringField(field.Name, value.GetValue<string>(), stored),
             "Edm.Int32" => new Int32Field(field.Name, value.GetValue<int>(), stored),
             "Edm.Int64" => new Int64Field(field.Name, value.GetValue<long>(), stored),
             "Edm.Double" => new DoubleField(field.Name, value.GetValue<double>(), stored),

AzureSearchEmulator/Indexing/UpsertIndexDocumentActionBase.cs

@@ -33,7 +33,8 @@ protected IEnumerable<IIndexableField> GetDocFields(SearchIndex index)
         return from f in index.Fields
             join v in Item on f.Name equals v.Key
             where v.Value != null
-            select f.CreateField(v.Value!); // [!]: null checked by where clause
+            from indexField in f.CreateFields(v.Value!) // [!]: null checked by where clause
+            select indexField;
     }
 
     protected static void MergeDocument(IndexingContext context, Term keyTerm, IEnumerable<IIndexableField> docFields, bool uploadIfMissing)
@@ -50,15 +51,13 @@ protected static void MergeDocument(IndexingContext context, Term keyTerm, IEnum
 
         var doc = docs.TotalHits == 0 ? new Document() : searcher.Doc(docs.ScoreDocs[0].Doc);
 
-        foreach (var docField in docFields)
+        var materialized = docFields.ToList();
+        foreach (var name in materialized.Select(f => f.Name).Distinct())
+        {
+            doc.RemoveFields(name);
+        }
+        foreach (var docField in materialized)
         {
-            var field = doc.GetField(docField.Name);
-
-            if (field != null)
-            {
-                doc.RemoveField(docField.Name);
-            }
-
             doc.Add(docField);
         }
 

AzureSearchEmulator/Searching/ODataQueryVisitor.cs

@@ -48,6 +48,7 @@ public Query Visit(BinaryOperatorToken tokenIn)
                 Right: LiteralToken literalToken
             })
         {
+            EnsureFilterable(path);
             return tokenIn.OperatorKind switch
             {
                 BinaryOperatorKind.Equal => HandleEqualComparison(path, literalToken),
@@ -67,6 +68,7 @@ public Query Visit(BinaryOperatorToken tokenIn)
                 Right: LiteralToken negatedLiteral
             })
         {
+            EnsureFilterable(negatedPath);
             var equalQuery = tokenIn.OperatorKind switch
             {
                 BinaryOperatorKind.Equal => HandleEqualComparison(negatedPath, negatedLiteral),
@@ -91,6 +93,17 @@ public Query Visit(BinaryOperatorToken tokenIn)
         throw new NotImplementedException();
     }
 
+    private void EnsureFilterable(string path)
+    {
+        if (_index is null) return;
+        var field = _index.Fields.FirstOrDefault(f => string.Equals(f.Name, path, StringComparison.OrdinalIgnoreCase));
+        if (field is null) return;
+        if (!field.Filterable)
+        {
+            throw new InvalidOperationException($"Field '{field.Name}' is not filterable.");
+        }
+    }
+
     private static Occur GetOccurFromOperator(BinaryOperatorKind operatorKind)
     {
         return operatorKind switch