Merge pull request #5616 from Particular/rhys-sql-testing

Add full text search to sql container

Author: rbev

README.md

@@ -52,6 +52,8 @@ Running all tests all the times takes a lot of resources. Tests are filtered bas
 - `MSMQ`
 - `RabbitMQ`
 - `SqlServer`
+- `SqlServerPersistence`
+- `PostgresSqlPersistence`
 - `SQS`
 
 NOTE: If no variable is defined all tests will be executed.
@@ -71,6 +73,7 @@ Local testing guides:
 - [Reverse Proxy Testing](docs/reverseproxy-testing.md)
 - [Forward Headers Testing](docs/forward-headers-testing.md)
 - [Authentication Testing](docs/authentication-testing.md)
+- [Persistence Tests](docs/testing-persistence.md)
 
 ## How to developer test the PowerShell Module
 

docs/testing-persistence.md

@@ -0,0 +1,38 @@
+# Local Testing of Persistence Providers
+
+ServiceControl supports multiple persistence types
+
+* RavenDB (default)
+* Microsoft SQL Server
+* PostgreSQL
+
+All persistence test projects can be run with `dotnet test` against the corresponding test project in `src/`.
+
+## RavenDB
+
+RavenDB persistence tests start an embedded RavenDB instance for the duration of the test run.
+
+## SQL Server
+
+SQL Server persistence tests use [Testcontainers](https://testcontainers.com/) and expect the local image
+`particular/servicecontrol-testing-sqlserver:latest`.
+
+Build that image locally before running SQL Server persistence tests:
+
+```shell
+docker buildx build --platform=linux/amd64 --tag particular/servicecontrol-testing-sqlserver:latest ./src/Scripts/Docker/servicecontrol-testing-sqlserver
+```
+
+If you want to use an existing SQL Server instance instead of a test container, set the `ServiceControl_Persistence_SqlServer_ConnectionString` environment variable to a valid SQL Server connection string.
+
+## PostgreSQL
+
+PostgreSQL persistence tests use [Testcontainers](https://testcontainers.com/) and start a `postgres:16-alpine` container automatically.
+
+If you want to use an existing PostgreSQL instance instead of a test container, set:
+
+```shell
+ServiceControl_Persistence_PostgreSql_ConnectionString
+```
+
+to a valid PostgreSQL connection string.

docs/testing.md

@@ -23,6 +23,7 @@ Installation engine tests run partial installations and checks:
 ## Persistence tests
 
 Persistence tests check assumptions at the persistence seam level by exercising each persister.
+For local setup details, see [Local Testing of Persistence Providers](testing-persistence.md).
 
 ## Transport tests
 

src/Scripts/Docker/servicecontrol-testing-sqlserver/Dockerfile

@@ -0,0 +1,21 @@
+FROM ubuntu:22.04
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install -y curl gnupg2 apt-transport-https && \
+    curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | \
+        gpg --dearmor -o /etc/apt/trusted.gpg.d/microsoft.gpg && \
+    curl -fsSL https://packages.microsoft.com/config/ubuntu/22.04/mssql-server-2025.list | \
+        tee /etc/apt/sources.list.d/mssql-server-2025.list && \
+    curl -fsSL https://packages.microsoft.com/config/ubuntu/22.04/prod.list | \
+        tee /etc/apt/sources.list.d/mssql-release.list && \
+    apt-get update && \
+    ACCEPT_EULA=Y apt-get install -y mssql-server mssql-server-fts && \
+    ACCEPT_EULA=Y apt-get install -y mssql-tools18 unixodbc-dev && \
+    # Clean up apt cache
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+EXPOSE 1433
+
+ENTRYPOINT [ "/opt/mssql/bin/sqlservr" ]

src/ServiceControl.Persistence.EFCore/Implementation/ExternalIntegrationRequestsDataStore.cs

@@ -5,15 +5,11 @@ namespace ServiceControl.Persistence.EFCore.Implementation;
 
 public class ExternalIntegrationRequestsDataStore : IExternalIntegrationRequestsDataStore, IHostedService
 {
-    public void Subscribe(Func<object[], Task> callback) =>
-        throw new NotImplementedException();
+    public void Subscribe(Func<object[], Task> callback) { }
 
-    public Task StoreDispatchRequest(IEnumerable<ExternalIntegrationDispatchRequest> dispatchRequests) =>
-        throw new NotImplementedException();
+    public Task StoreDispatchRequest(IEnumerable<ExternalIntegrationDispatchRequest> dispatchRequests) => Task.CompletedTask;
 
-    public Task StartAsync(CancellationToken cancellationToken) =>
-        throw new NotImplementedException();
+    public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask;
 
-    public Task StopAsync(CancellationToken cancellationToken) =>
-        throw new NotImplementedException();
+    public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
 }

src/ServiceControl.Persistence.EFCore/Implementation/FailedMessageViewIndexNotifications.cs

@@ -7,9 +7,7 @@ public class FailedMessageViewIndexNotifications : IFailedMessageViewIndexNotifi
     public IDisposable Subscribe(Func<FailedMessageTotals, Task> callback) =>
         throw new NotImplementedException();
 
-    public Task StartAsync(CancellationToken cancellationToken) =>
-        throw new NotImplementedException();
+    public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask;
 
-    public Task StopAsync(CancellationToken cancellationToken) =>
-        throw new NotImplementedException();
+    public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
 }

src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchTests.cs

@@ -0,0 +1,78 @@
+namespace ServiceControl.Persistence.Tests;
+
+using System;
+using System.Threading.Tasks;
+using EFCore.PostgreSql;
+using Npgsql;
+using NUnit.Framework;
+
+class FullTextSearchTests : PersistenceTestBase
+{
+    [Test]
+    public async Task Can_create_and_query_full_text_index()
+    {
+        var postgreSqlSettings = PersistenceSettings as PostgreSqlPersisterSettings;
+        Assert.That(postgreSqlSettings, Is.Not.Null);
+
+        var tableName = $"fts_{Guid.NewGuid():N}";
+        var commandTimeoutSeconds = 30;
+
+        await using var connection = new NpgsqlConnection(postgreSqlSettings.ConnectionString);
+        await connection.OpenAsync();
+
+        try
+        {
+            await ExecuteNonQuery(connection, $"""
+                CREATE TABLE public."{tableName}" (
+                    id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+                    body TEXT NOT NULL,
+                    search_vector tsvector GENERATED ALWAYS AS (to_tsvector('english', body)) STORED
+                )
+                """, commandTimeoutSeconds);
+
+            await ExecuteNonQuery(connection, $"""CREATE INDEX "{tableName}_search_vector_idx" ON public."{tableName}" USING GIN (search_vector)""", commandTimeoutSeconds);
+
+            await ExecuteNonQuery(connection, $"""
+                INSERT INTO public."{tableName}"(body) VALUES
+                ('quick brown fox jumps'),
+                ('azure service bus transport')
+                """, commandTimeoutSeconds);
+
+            var matchCount = await ExecuteScalarInt(connection, $"""
+                SELECT COUNT(1)
+                FROM public."{tableName}"
+                WHERE search_vector @@ plainto_tsquery('english', 'quick')
+                """, commandTimeoutSeconds);
+
+            Assert.That(matchCount, Is.EqualTo(1));
+        }
+        finally
+        {
+            await ExecuteNonQuery(connection, $"""DROP TABLE IF EXISTS public."{tableName}" CASCADE""", commandTimeoutSeconds, ignoreErrors: true);
+        }
+    }
+
+    static async Task<int> ExecuteScalarInt(NpgsqlConnection connection, string sql, int commandTimeoutSeconds)
+    {
+        await using var command = connection.CreateCommand();
+        command.CommandTimeout = commandTimeoutSeconds;
+        command.CommandText = sql;
+        var scalar = await command.ExecuteScalarAsync();
+        Assert.That(scalar, Is.Not.Null);
+        return Convert.ToInt32(scalar);
+    }
+
+    static async Task ExecuteNonQuery(NpgsqlConnection connection, string sql, int commandTimeoutSeconds, bool ignoreErrors = false)
+    {
+        try
+        {
+            await using var command = connection.CreateCommand();
+            command.CommandTimeout = commandTimeoutSeconds;
+            command.CommandText = sql;
+            await command.ExecuteNonQueryAsync();
+        }
+        catch when (ignoreErrors)
+        {
+        }
+    }
+}

src/ServiceControl.Persistence.Tests.PostgreSql/PostgreSqlSharedContainer.cs

@@ -7,6 +7,8 @@ namespace ServiceControl.Persistence.Tests;
 
 static class PostgreSqlSharedContainer
 {
+    const string docsPath = "docs/testing-persistence.md#postgresql";
+
     public static async Task<string> GetConnectionStringAsync(CancellationToken ct = default)
     {
         var envConnStr = Environment.GetEnvironmentVariable("ServiceControl_Persistence_PostgreSql_ConnectionString");
@@ -38,7 +40,17 @@ static async Task<PostgreSqlContainer> StartContainerAsync(CancellationToken ct)
     {
         var c = new PostgreSqlBuilder("postgres:16-alpine")
             .Build();
-        await c.StartAsync(ct);
+        try
+        {
+            await c.StartAsync(ct);
+        }
+        catch (Exception ex)
+        {
+            throw new InvalidOperationException(
+                $"Failed to start PostgreSQL persistence test container. See {docsPath} for setup instructions.",
+                ex);
+        }
+
         return c;
     }
 

src/ServiceControl.Persistence.Tests.PostgreSql/ServiceControl.Persistence.Tests.PostgreSql.csproj

@@ -26,6 +26,22 @@
 
   <ItemGroup>
     <Compile Include="..\ServiceControl.Persistence.Tests\**\*.cs" LinkBase="Shared" />
+
+    <!-- as features get implemented remove these exclusions -->
+    <Compile Remove="..\ServiceControl.Persistence.Tests\BodyStorage\AttachmentsBodyStorageTests.cs" />
+    <Compile Remove="..\ServiceControl.Persistence.Tests\Recoverability\EditHandlerAuditTests.cs" />
+    <Compile Remove="..\ServiceControl.Persistence.Tests\Recoverability\EditMessageTests.cs" />
+    <Compile Remove="..\ServiceControl.Persistence.Tests\Recoverability\RetryConfirmationProcessorTests.cs" />
+    <Compile Remove="..\ServiceControl.Persistence.Tests\Recoverability\ReturnToSenderDequeuerTests.cs" />
+    <Compile Remove="..\ServiceControl.Persistence.Tests\Throughput\AuditServiceMetadataTests.cs" />
+    <Compile Remove="..\ServiceControl.Persistence.Tests\Throughput\BrokerMetadataTests.cs" />
+    <Compile Remove="..\ServiceControl.Persistence.Tests\Throughput\EndpointsTests.cs" />
+    <Compile Remove="..\ServiceControl.Persistence.Tests\Throughput\ReportMasksTests.cs" />
+    <Compile Remove="..\ServiceControl.Persistence.Tests\CustomChecksDataStoreTests.cs" />
+    <Compile Remove="..\ServiceControl.Persistence.Tests\CustomCheckTests.cs" />
+    <Compile Remove="..\ServiceControl.Persistence.Tests\EnsureSettingsInContainer.cs" />
+    <Compile Remove="..\ServiceControl.Persistence.Tests\MonitoringDataStoreTests.cs" />
+    <Compile Remove="..\ServiceControl.Persistence.Tests\RetryStateTests.cs" />
   </ItemGroup>
 
 </Project>