From b21fe08d12a111fad6d04497b36823479d90cd7a Mon Sep 17 00:00:00 2001 From: Curt Hagenlocher Date: Sat, 18 Apr 2026 20:39:43 -0700 Subject: [PATCH 1/3] feat(csharp): make core ADBC and trace listeners AOT-compatible Adds net10.0 to the target frameworks for Apache.Arrow.Adbc and the trace-listener assembly, with true enabled on that TFM. - Replace FileVersionInfo.GetVersionInfo(assembly.Location) with AssemblyInformationalVersionAttribute lookup; Assembly.Location is empty under single-file publish. A guarded FileVersionInfo fallback is retained for JIT via RuntimeFeature.IsDynamicCodeSupported. - Rewrite IArrowArrayExtensions.SerializeToJson to dispatch through Utf8JsonWriter instead of the reflection-based JsonSerializer. Every value type ParseStructArray can produce is dispatched explicitly. SqlDecimal (Decimal128) now emits as a JSON number when the declared precision is <= 15 and as a string otherwise, replacing the previous accidental {IsNull, Value, Precision, ...} shape. - Rewrite FileListener.ActivityProcessor to serialize via a source- generated JsonSerializerContext. A new OtelAttributesConverter preserves OpenTelemetry-compatible attribute values (string, bool, int64, double, and homogeneous arrays) as native JSON; non-OTel values fall back to invariant-culture strings so the output doesn't drift by locale. - CAdbcDriverExporter.AdbcDriverInit returns NotImplemented rather than InternalError for unsupported ADBC versions so the importer's 1.1.0 -> 1.0.0 fallback works. - Add InternalsVisibleTo for Apache.Arrow.Adbc.Testing (the actual assembly name; the existing Apache.Arrow.Adbc.Tests entry is stale). Covered by 27 new golden-output tests for SerializeToJson and 14 new tests for OtelAttributesConverter (including invariant-culture verification under a non-English locale). --- .../Apache.Arrow.Adbc.csproj | 6 +- .../C/CAdbcDriverExporter.cs | 2 +- .../Extensions/IArrowArrayExtensions.cs | 116 +++++++- .../Properties/AssemblyInfo.cs | 1 + .../Tracing/ActivityTrace.cs | 27 +- ...row.Adbc.Telemetry.Traces.Listeners.csproj | 6 +- .../FileListener/ActivityProcessor.cs | 8 + .../FileListener/OtelAttributesConverter.cs | 204 +++++++++++++++ .../FileListener/SerializableActivity.cs | 3 + .../FileListener/TraceJsonContext.cs | 28 ++ .../SerializeStructToJsonTests.cs | 247 ++++++++++++++++++ .../OtelAttributesConverterTests.cs | 158 +++++++++++ 12 files changed, 800 insertions(+), 6 deletions(-) create mode 100644 csharp/src/Telemetry/Traces/Listeners/FileListener/OtelAttributesConverter.cs create mode 100644 csharp/src/Telemetry/Traces/Listeners/FileListener/TraceJsonContext.cs create mode 100644 csharp/test/Apache.Arrow.Adbc.Tests/SerializeStructToJsonTests.cs create mode 100644 csharp/test/Telemetry/Traces/Listeners/FileListener/OtelAttributesConverterTests.cs diff --git a/csharp/src/Apache.Arrow.Adbc/Apache.Arrow.Adbc.csproj b/csharp/src/Apache.Arrow.Adbc/Apache.Arrow.Adbc.csproj index 1a6e3ec653..5792b1b912 100644 --- a/csharp/src/Apache.Arrow.Adbc/Apache.Arrow.Adbc.csproj +++ b/csharp/src/Apache.Arrow.Adbc/Apache.Arrow.Adbc.csproj @@ -1,10 +1,14 @@ - netstandard2.0;net8.0 + netstandard2.0;net8.0;net10.0 true readme.md + + + true + diff --git a/csharp/src/Apache.Arrow.Adbc/C/CAdbcDriverExporter.cs b/csharp/src/Apache.Arrow.Adbc/C/CAdbcDriverExporter.cs index 0accb13381..9b6a0c7147 100644 --- a/csharp/src/Apache.Arrow.Adbc/C/CAdbcDriverExporter.cs +++ b/csharp/src/Apache.Arrow.Adbc/C/CAdbcDriverExporter.cs @@ -98,7 +98,7 @@ public unsafe static AdbcStatusCode AdbcDriverInit(int version, CAdbcDriver* nat if (version != AdbcVersion.Version_1_0_0) { // TODO: implement support for AdbcVersion.Version_1_1_0 - return AdbcStatusCode.InternalError; + return AdbcStatusCode.NotImplemented; } DriverStub stub = new DriverStub(driver); diff --git a/csharp/src/Apache.Arrow.Adbc/Extensions/IArrowArrayExtensions.cs b/csharp/src/Apache.Arrow.Adbc/Extensions/IArrowArrayExtensions.cs index fab63f4ef1..2eaf75a9c8 100644 --- a/csharp/src/Apache.Arrow.Adbc/Extensions/IArrowArrayExtensions.cs +++ b/csharp/src/Apache.Arrow.Adbc/Extensions/IArrowArrayExtensions.cs @@ -317,7 +317,121 @@ private static string SerializeToJson(StructArray structArray, int index) { Dictionary? obj = ParseStructArray(structArray, index); - return JsonSerializer.Serialize(obj); + using MemoryStream ms = new(); + using (Utf8JsonWriter writer = new(ms)) + { + WriteJsonValue(writer, obj); + } + return System.Text.Encoding.UTF8.GetString(ms.ToArray()); + } + + private static void WriteJsonValue(Utf8JsonWriter writer, object? value) + { + switch (value) + { + case null: + writer.WriteNullValue(); + return; + case string s: + writer.WriteStringValue(s); + return; + case bool b: + writer.WriteBooleanValue(b); + return; + case int i: + writer.WriteNumberValue(i); + return; + case long l: + writer.WriteNumberValue(l); + return; + case short sh: + writer.WriteNumberValue(sh); + return; + case byte by: + writer.WriteNumberValue(by); + return; + case sbyte sb: + writer.WriteNumberValue(sb); + return; + case uint ui: + writer.WriteNumberValue(ui); + return; + case ulong ul: + writer.WriteNumberValue(ul); + return; + case ushort us: + writer.WriteNumberValue(us); + return; + case float f: + writer.WriteNumberValue(f); + return; + case double d: + writer.WriteNumberValue(d); + return; + case decimal dec: + writer.WriteNumberValue(dec); + return; + case SqlDecimal sqlDec: + // Decimal32/Decimal64 come back as plain `decimal` and are handled above. + // Decimal128 values arrive here; SqlDecimal.Precision reflects the column's + // declared precision (every row from the same column shares it, so schema + // stays stable). Any decimal that fits in 15 significant digits round-trips + // cleanly through double, so it's safe to emit as a JSON number. Wider + // columns are emitted as strings — consumers must parse them deliberately. + if (sqlDec.Precision <= 15) + { + writer.WriteNumberValue(sqlDec.Value); + } + else + { + // SqlDecimal.ToString() formats from the Data array, so it handles + // precisions that would overflow SqlDecimal.Value (>28 digits). + writer.WriteStringValue(sqlDec.ToString()); + } + return; + case DateTime dt: + writer.WriteStringValue(dt); + return; + case DateTimeOffset dto: + writer.WriteStringValue(dto); + return; +#if NET6_0_OR_GREATER + case TimeOnly time: + // Match System.Text.Json's ISO-ish "HH:mm:ss[.fraction]" format. + // FFFFFFF elides trailing zeros and the decimal point when zero. + writer.WriteStringValue(time.ToString("HH:mm:ss.FFFFFFF", System.Globalization.CultureInfo.InvariantCulture)); + return; + case DateOnly date: + writer.WriteStringValue(date.ToString("yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture)); + return; +#endif + case Guid g: + writer.WriteStringValue(g); + return; + case byte[] bytes: + writer.WriteBase64StringValue(bytes); + return; + case IDictionary dict: + writer.WriteStartObject(); + foreach (KeyValuePair kv in dict) + { + writer.WritePropertyName(kv.Key); + WriteJsonValue(writer, kv.Value); + } + writer.WriteEndObject(); + return; + case IEnumerable enumerable: + writer.WriteStartArray(); + foreach (object? item in enumerable) + { + WriteJsonValue(writer, item); + } + writer.WriteEndArray(); + return; + default: + writer.WriteStringValue(value.ToString()); + return; + } } /// diff --git a/csharp/src/Apache.Arrow.Adbc/Properties/AssemblyInfo.cs b/csharp/src/Apache.Arrow.Adbc/Properties/AssemblyInfo.cs index f9fe442ad2..dc9da60373 100644 --- a/csharp/src/Apache.Arrow.Adbc/Properties/AssemblyInfo.cs +++ b/csharp/src/Apache.Arrow.Adbc/Properties/AssemblyInfo.cs @@ -19,3 +19,4 @@ [assembly: InternalsVisibleTo("Apache.Arrow.Adbc.Drivers.BigQuery, PublicKey=0024000004800000940000000602000000240000525341310004000001000100e504183f6d470d6b67b6d19212be3e1f598f70c246a120194bc38130101d0c1853e4a0f2232cb12e37a7a90e707aabd38511dac4f25fcb0d691b2aa265900bf42de7f70468fc997551a40e1e0679b605aa2088a4a69e07c117e988f5b1738c570ee66997fba02485e7856a49eca5fd0706d09899b8312577cbb9034599fc92d4")] [assembly: InternalsVisibleTo("Apache.Arrow.Adbc.Tests.Drivers.Interop.FlightSql, PublicKey=0024000004800000940000000602000000240000525341310004000001000100e504183f6d470d6b67b6d19212be3e1f598f70c246a120194bc38130101d0c1853e4a0f2232cb12e37a7a90e707aabd38511dac4f25fcb0d691b2aa265900bf42de7f70468fc997551a40e1e0679b605aa2088a4a69e07c117e988f5b1738c570ee66997fba02485e7856a49eca5fd0706d09899b8312577cbb9034599fc92d4")] [assembly: InternalsVisibleTo("Apache.Arrow.Adbc.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100e504183f6d470d6b67b6d19212be3e1f598f70c246a120194bc38130101d0c1853e4a0f2232cb12e37a7a90e707aabd38511dac4f25fcb0d691b2aa265900bf42de7f70468fc997551a40e1e0679b605aa2088a4a69e07c117e988f5b1738c570ee66997fba02485e7856a49eca5fd0706d09899b8312577cbb9034599fc92d4")] +[assembly: InternalsVisibleTo("Apache.Arrow.Adbc.Testing, PublicKey=0024000004800000940000000602000000240000525341310004000001000100e504183f6d470d6b67b6d19212be3e1f598f70c246a120194bc38130101d0c1853e4a0f2232cb12e37a7a90e707aabd38511dac4f25fcb0d691b2aa265900bf42de7f70468fc997551a40e1e0679b605aa2088a4a69e07c117e988f5b1738c570ee66997fba02485e7856a49eca5fd0706d09899b8312577cbb9034599fc92d4")] diff --git a/csharp/src/Apache.Arrow.Adbc/Tracing/ActivityTrace.cs b/csharp/src/Apache.Arrow.Adbc/Tracing/ActivityTrace.cs index b332590a56..422028094b 100644 --- a/csharp/src/Apache.Arrow.Adbc/Tracing/ActivityTrace.cs +++ b/csharp/src/Apache.Arrow.Adbc/Tracing/ActivityTrace.cs @@ -18,6 +18,8 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Reflection; using System.Runtime.CompilerServices; using System.Threading.Tasks; @@ -40,8 +42,7 @@ public sealed class ActivityTrace : IDisposable public ActivityTrace(string? activitySourceName = default, string? activitySourceVersion = default, string? traceParent = default, IEnumerable>? tags = default) { activitySourceName ??= GetType().Assembly.GetName().Name!; - // It's okay to have a null version. - activitySourceVersion ??= FileVersionInfo.GetVersionInfo(GetType().Assembly.Location).ProductVersion; + activitySourceVersion ??= GetAssemblyVersion(typeof(ActivityTrace)); if (string.IsNullOrWhiteSpace(activitySourceName)) { throw new ArgumentNullException(nameof(activitySourceName)); @@ -263,6 +264,28 @@ public void Dispose() ActivitySource.Dispose(); } + /// + /// If possible, gets the file version for the assembly associated with the given Type. + /// + [SuppressMessage("SingleFile", "IL3000", Justification="Using guard")] + public static string GetAssemblyVersion(Type type) + { + var versionAttr = type.Assembly.GetCustomAttribute(); + if (versionAttr?.InformationalVersion != null) return versionAttr.InformationalVersion; + +#if NET8_0_OR_GREATER + if (RuntimeFeature.IsDynamicCodeSupported && type.Assembly.Location != null) + { + var fileVersion = FileVersionInfo.GetVersionInfo(type.Assembly.Location).ProductVersion; + if (fileVersion != null) return fileVersion; + } +#endif + + string? assemblyVersion = type.Assembly.GetName().Version?.ToString(); + + return assemblyVersion ?? string.Empty; + } + private static void WriteTraceException(Exception exception, Activity? activity) { activity?.AddException(exception); diff --git a/csharp/src/Telemetry/Traces/Listeners/Apache.Arrow.Adbc.Telemetry.Traces.Listeners.csproj b/csharp/src/Telemetry/Traces/Listeners/Apache.Arrow.Adbc.Telemetry.Traces.Listeners.csproj index d45d327253..34e77b48e0 100644 --- a/csharp/src/Telemetry/Traces/Listeners/Apache.Arrow.Adbc.Telemetry.Traces.Listeners.csproj +++ b/csharp/src/Telemetry/Traces/Listeners/Apache.Arrow.Adbc.Telemetry.Traces.Listeners.csproj @@ -1,10 +1,14 @@ - netstandard2.0;net8.0 + netstandard2.0;net8.0;net10.0 enable + + true + + diff --git a/csharp/src/Telemetry/Traces/Listeners/FileListener/ActivityProcessor.cs b/csharp/src/Telemetry/Traces/Listeners/FileListener/ActivityProcessor.cs index 20fff58408..00f6516dbd 100644 --- a/csharp/src/Telemetry/Traces/Listeners/FileListener/ActivityProcessor.cs +++ b/csharp/src/Telemetry/Traces/Listeners/FileListener/ActivityProcessor.cs @@ -89,9 +89,17 @@ private async Task ProcessActivitiesAsync(CancellationToken cancellationToken) stream.SetLength(0); SerializableActivity serializableActivity = new(activity); +#if NET6_0_OR_GREATER + await JsonSerializer.SerializeAsync( + stream, + serializableActivity, + TraceJsonContext.Default.SerializableActivity, + cancellationToken).ConfigureAwait(false); +#else await JsonSerializer.SerializeAsync( stream, serializableActivity, cancellationToken: cancellationToken).ConfigureAwait(false); +#endif stream.Write(s_newLine, 0, s_newLine.Length); stream.Position = 0; diff --git a/csharp/src/Telemetry/Traces/Listeners/FileListener/OtelAttributesConverter.cs b/csharp/src/Telemetry/Traces/Listeners/FileListener/OtelAttributesConverter.cs new file mode 100644 index 0000000000..4bc9b4fa6c --- /dev/null +++ b/csharp/src/Telemetry/Traces/Listeners/FileListener/OtelAttributesConverter.cs @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Apache.Arrow.Adbc.Telemetry.Traces.Listeners.FileListener +{ + /// + /// Writes OTel-compatible attribute values as native JSON. The OpenTelemetry spec allows + /// attribute values to be string, bool, long, or double, or + /// homogeneous arrays of those types. Values of those types are emitted as native JSON + /// scalars/arrays. Anything else (DateTime, custom types, etc.) falls through to an + /// invariant-culture string representation so the serialized form doesn't vary by locale. + /// + internal static class OtelAttributeWriter + { + public static void WriteValue(Utf8JsonWriter writer, object? value) + { + switch (value) + { + case null: + writer.WriteNullValue(); + return; + case string s: + writer.WriteStringValue(s); + return; + case bool b: + writer.WriteBooleanValue(b); + return; + // OTel's integer attributes are int64. Any integral type up to long fits. + case long l: + writer.WriteNumberValue(l); + return; + case int i: + writer.WriteNumberValue(i); + return; + case short sh: + writer.WriteNumberValue(sh); + return; + case byte by: + writer.WriteNumberValue(by); + return; + case sbyte sb: + writer.WriteNumberValue(sb); + return; + case uint ui: + writer.WriteNumberValue(ui); + return; + case ulong ul: + writer.WriteNumberValue(ul); + return; + case ushort us: + writer.WriteNumberValue(us); + return; + // OTel floating-point is double; float widens without loss. + case double d: + writer.WriteNumberValue(d); + return; + case float f: + writer.WriteNumberValue(f); + return; + // Homogeneous OTel arrays. Iterate with the concrete element type so each + // scalar takes the native-JSON path rather than the fallback. + case string?[] ss: + WriteArray(writer, ss); + return; + case bool[] bs: + WriteArray(writer, bs); + return; + case long[] ls: + WriteArray(writer, ls); + return; + case int[] iarr: + WriteArray(writer, iarr); + return; + case double[] ds: + WriteArray(writer, ds); + return; + // Generic enumerable fallback (covers ImmutableArray, List, etc.) — + // each element is dispatched recursively. + case IEnumerable enumerable: + writer.WriteStartArray(); + foreach (object? item in enumerable) + { + WriteValue(writer, item); + } + writer.WriteEndArray(); + return; + // Non-OTel types: invariant-culture string. This keeps the output + // portable across locales for things like DateTime or custom structs. + case IFormattable formattable: + writer.WriteStringValue(formattable.ToString(null, CultureInfo.InvariantCulture)); + return; + default: + writer.WriteStringValue(value.ToString()); + return; + } + } + + private static void WriteArray(Utf8JsonWriter writer, string?[] array) + { + writer.WriteStartArray(); + foreach (string? item in array) + { + if (item == null) { writer.WriteNullValue(); } else { writer.WriteStringValue(item); } + } + writer.WriteEndArray(); + } + + private static void WriteArray(Utf8JsonWriter writer, bool[] array) + { + writer.WriteStartArray(); + foreach (bool item in array) { writer.WriteBooleanValue(item); } + writer.WriteEndArray(); + } + + private static void WriteArray(Utf8JsonWriter writer, long[] array) + { + writer.WriteStartArray(); + foreach (long item in array) { writer.WriteNumberValue(item); } + writer.WriteEndArray(); + } + + private static void WriteArray(Utf8JsonWriter writer, int[] array) + { + writer.WriteStartArray(); + foreach (int item in array) { writer.WriteNumberValue(item); } + writer.WriteEndArray(); + } + + private static void WriteArray(Utf8JsonWriter writer, double[] array) + { + writer.WriteStartArray(); + foreach (double item in array) { writer.WriteNumberValue(item); } + writer.WriteEndArray(); + } + } + + /// + /// Write-only converter for and similar + /// IReadOnlyDictionary<string, object?> properties. Emits a JSON object. + /// + internal sealed class OtelAttributesDictionaryConverter : JsonConverter> + { + public override IReadOnlyDictionary? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + => throw new NotSupportedException("Reading SerializableActivity from JSON is not supported."); + + public override void Write(Utf8JsonWriter writer, IReadOnlyDictionary value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + foreach (KeyValuePair kv in value) + { + writer.WritePropertyName(kv.Key); + OtelAttributeWriter.WriteValue(writer, kv.Value); + } + writer.WriteEndObject(); + } + } + + /// + /// Write-only converter for IReadOnlyList<KeyValuePair<string, object?>> + /// properties (ActivityEvent.Tags, ActivityLink.Tags). Emits a JSON array of + /// {"Key": ..., "Value": ...} objects to match the shape produced by the + /// previous reflection-based JsonSerializer. + /// + internal sealed class OtelAttributesListConverter : JsonConverter>> + { + public override IReadOnlyList>? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + => throw new NotSupportedException("Reading SerializableActivity from JSON is not supported."); + + public override void Write(Utf8JsonWriter writer, IReadOnlyList> value, JsonSerializerOptions options) + { + writer.WriteStartArray(); + foreach (KeyValuePair kv in value) + { + writer.WriteStartObject(); + writer.WriteString("Key", kv.Key); + writer.WritePropertyName("Value"); + OtelAttributeWriter.WriteValue(writer, kv.Value); + writer.WriteEndObject(); + } + writer.WriteEndArray(); + } + } +} diff --git a/csharp/src/Telemetry/Traces/Listeners/FileListener/SerializableActivity.cs b/csharp/src/Telemetry/Traces/Listeners/FileListener/SerializableActivity.cs index 2af071a681..c3465861cc 100644 --- a/csharp/src/Telemetry/Traces/Listeners/FileListener/SerializableActivity.cs +++ b/csharp/src/Telemetry/Traces/Listeners/FileListener/SerializableActivity.cs @@ -122,6 +122,7 @@ internal SerializableActivity(Activity activity) : this( public string? ParentSpanId { get; set; } public string? IdFormat { get; set; } + [JsonConverter(typeof(OtelAttributesDictionaryConverter))] public IReadOnlyDictionary TagObjects { get; set; } = new Dictionary(); public IReadOnlyList Events { get; set; } = []; public IReadOnlyList Links { get; set; } = []; @@ -140,6 +141,7 @@ internal class SerializableActivityEvent /// public DateTimeOffset Timestamp { get; set; } + [JsonConverter(typeof(OtelAttributesListConverter))] public IReadOnlyList> Tags { get; set; } = []; public static implicit operator SerializableActivityEvent(ActivityEvent source) @@ -157,6 +159,7 @@ internal class SerializableActivityLink { public SerializableActivityContext? Context { get; set; } + [JsonConverter(typeof(OtelAttributesListConverter))] public IReadOnlyList>? Tags { get; set; } = []; public static implicit operator SerializableActivityLink(ActivityLink source) diff --git a/csharp/src/Telemetry/Traces/Listeners/FileListener/TraceJsonContext.cs b/csharp/src/Telemetry/Traces/Listeners/FileListener/TraceJsonContext.cs new file mode 100644 index 0000000000..ae0c0d7ba6 --- /dev/null +++ b/csharp/src/Telemetry/Traces/Listeners/FileListener/TraceJsonContext.cs @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if NET6_0_OR_GREATER +using System.Text.Json.Serialization; + +namespace Apache.Arrow.Adbc.Telemetry.Traces.Listeners.FileListener +{ + [JsonSerializable(typeof(SerializableActivity))] + internal partial class TraceJsonContext : JsonSerializerContext + { + } +} +#endif diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/SerializeStructToJsonTests.cs b/csharp/test/Apache.Arrow.Adbc.Tests/SerializeStructToJsonTests.cs new file mode 100644 index 0000000000..d5ff7d0bca --- /dev/null +++ b/csharp/test/Apache.Arrow.Adbc.Tests/SerializeStructToJsonTests.cs @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Data.SqlTypes; +using Apache.Arrow.Adbc.Extensions; +using Apache.Arrow.Types; +using Xunit; + +namespace Apache.Arrow.Adbc.Tests +{ + /// + /// Golden-output tests for IArrowArrayExtensions.ValueAt(index, StructResultType.JsonString) + /// — i.e. the private SerializeToJson path. Each test builds a single-row + /// containing one field of a particular Arrow type, runs it through the JSON conversion, and asserts + /// the exact string output. + /// + public class SerializeStructToJsonTests + { + private static string SerializeRow(StructArray structArray, int index = 0) + => (string)structArray.ValueAt(index, StructResultType.JsonString)!; + + private static StructArray SingleFieldStruct(string fieldName, IArrowType type, IArrowArray values) + { + var structType = new StructType(new[] { new Field(fieldName, type, nullable: true) }); + return new StructArray(structType, values.Length, new[] { values }, new ArrowBuffer.BitmapBuilder().Build()); + } + + [Fact] + public void Bool() + { + var values = new BooleanArray.Builder().Append(true).Build(); + var s = SingleFieldStruct("f", BooleanType.Default, values); + Assert.Equal("{\"f\":true}", SerializeRow(s)); + } + + [Fact] + public void Int8() => AssertJson("{\"f\":-7}", Int8Type.Default, new Int8Array.Builder().Append((sbyte)-7).Build()); + + [Fact] + public void Int16() => AssertJson("{\"f\":-300}", Int16Type.Default, new Int16Array.Builder().Append((short)-300).Build()); + + [Fact] + public void Int32() => AssertJson("{\"f\":-70000}", Int32Type.Default, new Int32Array.Builder().Append(-70000).Build()); + + [Fact] + public void Int64() => AssertJson("{\"f\":-5000000000}", Int64Type.Default, new Int64Array.Builder().Append(-5_000_000_000L).Build()); + + [Fact] + public void UInt8() => AssertJson("{\"f\":200}", UInt8Type.Default, new UInt8Array.Builder().Append((byte)200).Build()); + + [Fact] + public void UInt16() => AssertJson("{\"f\":60000}", UInt16Type.Default, new UInt16Array.Builder().Append((ushort)60000).Build()); + + [Fact] + public void UInt32() => AssertJson("{\"f\":4000000000}", UInt32Type.Default, new UInt32Array.Builder().Append(4_000_000_000u).Build()); + + [Fact] + public void UInt64() => AssertJson("{\"f\":9000000000000000000}", UInt64Type.Default, new UInt64Array.Builder().Append(9_000_000_000_000_000_000ul).Build()); + + [Fact] + public void Float() => AssertJson("{\"f\":1.5}", FloatType.Default, new FloatArray.Builder().Append(1.5f).Build()); + + [Fact] + public void Double() => AssertJson("{\"f\":3.25}", DoubleType.Default, new DoubleArray.Builder().Append(3.25).Build()); + + [Fact] + public void String() => AssertJson("{\"f\":\"hi\"}", StringType.Default, new StringArray.Builder().Append("hi").Build()); + + [Fact] + public void StringWithEscapes() + // JsonSerializer / Utf8JsonWriter default encoder escapes '"' as \u0022 + // (relaxed encoder would produce \"). + => AssertJson("{\"f\":\"a\\u0022b\\nc\"}", StringType.Default, new StringArray.Builder().Append("a\"b\nc").Build()); + + [Fact] + public void Binary() + { + var values = new BinaryArray.Builder().Append(new byte[] { 0x01, 0x02, 0x03 }).Build(); + AssertJson("{\"f\":\"AQID\"}", BinaryType.Default, values); + } + + [Fact] + public void Date32() + { + var d = new DateTime(2026, 4, 18, 0, 0, 0, DateTimeKind.Unspecified); + var values = new Date32Array.Builder().Append(d).Build(); + AssertJson("{\"f\":\"2026-04-18T00:00:00\"}", Date32Type.Default, values); + } + + [Fact] + public void Date64() + { + var d = new DateTime(2026, 4, 18, 0, 0, 0, DateTimeKind.Unspecified); + var values = new Date64Array.Builder().Append(d).Build(); + AssertJson("{\"f\":\"2026-04-18T00:00:00\"}", Date64Type.Default, values); + } + + [Fact] + public void Timestamp() + { + var ts = new DateTimeOffset(2026, 4, 18, 12, 34, 56, TimeSpan.Zero); + var values = new TimestampArray.Builder(TimestampType.Default).Append(ts).Build(); + AssertJson("{\"f\":\"2026-04-18T12:34:56+00:00\"}", TimestampType.Default, values); + } + + [Fact] + public void Time32Seconds() + { + // 5 minutes past midnight + var builder = new Time32Array.Builder(TimeUnit.Second); + builder.Append(300); + AssertJson("{\"f\":\"00:05:00\"}", new Time32Type(TimeUnit.Second), builder.Build()); + } + + [Fact] + public void Time64Microseconds() + { + var builder = new Time64Array.Builder(TimeUnit.Microsecond); + builder.Append(1_000_000L); // 1 second + AssertJson("{\"f\":\"00:00:01\"}", new Time64Type(TimeUnit.Microsecond), builder.Build()); + } + + [Fact] + public void Decimal32_AsNumber() + { + var type = new Decimal32Type(9, 2); + var builder = new Decimal32Array.Builder(type); + builder.Append(12.34m); + AssertJson("{\"f\":12.34}", type, builder.Build()); + } + + [Fact] + public void Decimal64_AsNumber() + { + var type = new Decimal64Type(15, 3); + var builder = new Decimal64Array.Builder(type); + builder.Append(123456789.012m); + AssertJson("{\"f\":123456789.012}", type, builder.Build()); + } + + [Fact] + public void Decimal128_NarrowPrecision_AsNumber() + { + // Precision <= 15 — every value fits in a double-round-tripping decimal, + // so we emit a bare JSON number. + var type = new Decimal128Type(10, 2); + var builder = new Decimal128Array.Builder(type); + builder.Append(123.45m); + AssertJson("{\"f\":123.45}", type, builder.Build()); + } + + [Fact] + public void Decimal128_WidePrecision_AsString() + { + // Precision > 15 — values may not round-trip through double. Emit as a JSON + // string; consumers who care about precision parse it with a decimal type. + var type = new Decimal128Type(20, 2); + var builder = new Decimal128Array.Builder(type); + builder.Append(12345678901234567.89m); + AssertJson("{\"f\":\"12345678901234567.89\"}", type, builder.Build()); + } + + [Fact] + public void Decimal256_AsString() + { + // ValueAt returns Decimal256 values as strings (GetString), so the JSON is a string. + var type = new Decimal256Type(30, 2); + var builder = new Decimal256Array.Builder(type); + builder.Append(9999.99m); + AssertJson("{\"f\":\"9999.99\"}", type, builder.Build()); + } + + [Fact] + public void NullField() + { + // Build an Int32Array with a single null entry. + var builder = new Int32Array.Builder(); + builder.AppendNull(); + AssertJson("{\"f\":null}", Int32Type.Default, builder.Build()); + } + + [Fact] + public void ListOfInt32() + { + var listBuilder = new ListArray.Builder(Int32Type.Default); + var valuesBuilder = (Int32Array.Builder)listBuilder.ValueBuilder; + listBuilder.Append(); + valuesBuilder.AppendRange(new[] { 1, 2, 3 }); + ListArray list = listBuilder.Build(); + + var s = SingleFieldStruct("f", new ListType(Int32Type.Default), list); + Assert.Equal("{\"f\":[1,2,3]}", SerializeRow(s)); + } + + [Fact] + public void NestedStruct() + { + // outer struct containing an inner struct with two int fields + var innerA = new Int32Array.Builder().Append(10).Build(); + var innerB = new StringArray.Builder().Append("ten").Build(); + + var innerType = new StructType(new[] + { + new Field("a", Int32Type.Default, nullable: true), + new Field("b", StringType.Default, nullable: true), + }); + + var innerStruct = new StructArray( + innerType, 1, + new IArrowArray[] { innerA, innerB }, + new ArrowBuffer.BitmapBuilder().Build()); + + var outerType = new StructType(new[] { new Field("inner", innerType, nullable: true) }); + var outer = new StructArray( + outerType, 1, + new IArrowArray[] { innerStruct }, + new ArrowBuffer.BitmapBuilder().Build()); + + string actual = SerializeRow(outer); + Assert.True("{\"inner\":{\"a\":10,\"b\":\"ten\"}}" == actual, $"actual: {actual}"); + } + + private static void AssertJson(string expected, IArrowType type, IArrowArray values) + { + var s = SingleFieldStruct("f", type, values); + string actual = SerializeRow(s); + // Untruncated failure output so golden mismatches are actionable. + Assert.True(expected == actual, $"\nexpected: {expected}\nactual: {actual}"); + } + } +} diff --git a/csharp/test/Telemetry/Traces/Listeners/FileListener/OtelAttributesConverterTests.cs b/csharp/test/Telemetry/Traces/Listeners/FileListener/OtelAttributesConverterTests.cs new file mode 100644 index 0000000000..9ba0434b2e --- /dev/null +++ b/csharp/test/Telemetry/Traces/Listeners/FileListener/OtelAttributesConverterTests.cs @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; +using System.Text.Json; +using Apache.Arrow.Adbc.Telemetry.Traces.Listeners.FileListener; + +namespace Apache.Arrow.Adbc.Tests.Telemetry.Traces.Listeners.FileListener +{ + /// + /// Verifies that emits OTel-compatible types as native + /// JSON scalars/arrays and falls back to an invariant-culture string for everything else. + /// + public class OtelAttributesConverterTests + { + private static string WriteDict(IReadOnlyDictionary dict) + { + var converter = new OtelAttributesDictionaryConverter(); + using var ms = new MemoryStream(); + using (var writer = new Utf8JsonWriter(ms)) + { + converter.Write(writer, dict, new JsonSerializerOptions()); + } + return Encoding.UTF8.GetString(ms.ToArray()); + } + + private static string WriteList(IReadOnlyList> list) + { + var converter = new OtelAttributesListConverter(); + using var ms = new MemoryStream(); + using (var writer = new Utf8JsonWriter(ms)) + { + converter.Write(writer, list, new JsonSerializerOptions()); + } + return Encoding.UTF8.GetString(ms.ToArray()); + } + + [Fact] + public void StringValueIsNativeString() + => Assert.Equal("{\"k\":\"v\"}", WriteDict(new Dictionary { ["k"] = "v" })); + + [Fact] + public void BoolValueIsNativeBoolean() + => Assert.Equal("{\"k\":true}", WriteDict(new Dictionary { ["k"] = true })); + + [Fact] + public void Int32ValueIsNativeNumber() + => Assert.Equal("{\"k\":42}", WriteDict(new Dictionary { ["k"] = 42 })); + + [Fact] + public void Int64ValueIsNativeNumber() + => Assert.Equal("{\"k\":5000000000}", WriteDict(new Dictionary { ["k"] = 5_000_000_000L })); + + [Fact] + public void DoubleValueIsNativeNumber() + => Assert.Equal("{\"k\":1.5}", WriteDict(new Dictionary { ["k"] = 1.5 })); + + [Fact] + public void NullValueIsJsonNull() + => Assert.Equal("{\"k\":null}", WriteDict(new Dictionary { ["k"] = null })); + + [Fact] + public void StringArrayIsJsonArrayOfStrings() + => Assert.Equal("{\"k\":[\"a\",\"b\"]}", WriteDict(new Dictionary { ["k"] = new[] { "a", "b" } })); + + [Fact] + public void BoolArrayIsJsonArrayOfBooleans() + => Assert.Equal("{\"k\":[true,false]}", WriteDict(new Dictionary { ["k"] = new[] { true, false } })); + + [Fact] + public void Int64ArrayIsJsonArrayOfNumbers() + => Assert.Equal("{\"k\":[1,2,3]}", WriteDict(new Dictionary { ["k"] = new long[] { 1, 2, 3 } })); + + [Fact] + public void DoubleArrayIsJsonArrayOfNumbers() + => Assert.Equal("{\"k\":[1.5,2.5]}", WriteDict(new Dictionary { ["k"] = new[] { 1.5, 2.5 } })); + + [Fact] + public void DateTimeFallsBackToInvariantString() + { + // Explicitly run under a comma-as-decimal locale to prove the output is invariant. + var prev = CultureInfo.CurrentCulture; + try + { + CultureInfo.CurrentCulture = new CultureInfo("de-DE"); + var dt = new DateTime(2026, 4, 18, 12, 34, 56, DateTimeKind.Utc); + string actual = WriteDict(new Dictionary { ["k"] = dt }); + // IFormattable.ToString(null, invariant) for DateTime uses the invariant + // general format: "04/18/2026 12:34:56". + Assert.Equal("{\"k\":\"04/18/2026 12:34:56\"}", actual); + } + finally + { + CultureInfo.CurrentCulture = prev; + } + } + + [Fact] + public void DecimalFallsBackToInvariantNumericString() + { + var prev = CultureInfo.CurrentCulture; + try + { + CultureInfo.CurrentCulture = new CultureInfo("de-DE"); + // decimal isn't in the OTel spec — falls to invariant string. Period, not comma. + string actual = WriteDict(new Dictionary { ["k"] = 1.5m }); + Assert.Equal("{\"k\":\"1.5\"}", actual); + } + finally + { + CultureInfo.CurrentCulture = prev; + } + } + + [Fact] + public void ListConverterWritesArrayOfKeyValueObjects() + { + var list = new[] + { + new KeyValuePair("a", 1), + new KeyValuePair("b", "two"), + new KeyValuePair("c", true), + }; + Assert.Equal("[{\"Key\":\"a\",\"Value\":1},{\"Key\":\"b\",\"Value\":\"two\"},{\"Key\":\"c\",\"Value\":true}]", WriteList(list)); + } + + [Fact] + public void ReadThrows() + { + var json = "{\"k\":1}"; + byte[] bytes = Encoding.UTF8.GetBytes(json); + var options = new JsonSerializerOptions(); + Assert.Throws(() => + { + var reader = new Utf8JsonReader(bytes); + new OtelAttributesDictionaryConverter().Read(ref reader, typeof(IReadOnlyDictionary), options); + }); + } + } +} From bd015cdc177f7b4c5da274334be74d412f096f93 Mon Sep 17 00:00:00 2001 From: Curt Hagenlocher Date: Sun, 19 Apr 2026 06:15:49 -0700 Subject: [PATCH 2/3] Make changes from Copilot PR feedback and add a round-trip test inside C# with a test fixture driver. --- .github/workflows/csharp.yml | 50 +++- .../C/CAdbcDriverExporter.cs | 4 +- .../C/CAdbcDriverImporter.Defaults.cs | 2 +- .../C/CAdbcDriverImporter.cs | 62 +++-- csharp/src/Apache.Arrow.Adbc/C/Delegates.cs | 3 +- .../Tracing/ActivityTrace.cs | 2 +- .../FileListener/OtelAttributesConverter.cs | 10 +- .../Apache.Arrow.Adbc.Testing.csproj | 1 + .../ExportedDriverRoundTripTests.cs | 252 ++++++++++++++++++ 9 files changed, 351 insertions(+), 35 deletions(-) create mode 100644 csharp/test/Apache.Arrow.Adbc.Tests/ExportedDriverRoundTripTests.cs diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml index 71c9392dd6..ffd740e33f 100644 --- a/.github/workflows/csharp.yml +++ b/.github/workflows/csharp.yml @@ -50,7 +50,7 @@ jobs: strategy: fail-fast: false matrix: - dotnet: ['8.0.x'] + dotnet: ['8.0.x', '10.0.x'] os: [ubuntu-latest, windows-2022, macos-15-intel, macos-latest] steps: - name: Install C# @@ -68,3 +68,51 @@ jobs: - name: Test shell: bash run: ci/scripts/csharp_test.sh $(pwd) + + # TODO: Create a test fixture driver to use for interop testing as the + # real drivers have migrated to https://github.com/adbc-drivers + + # Publishes the Apache driver as a NativeAOT shared library (net10) and + # loads it from Python via adbc_driver_manager to catch AOT regressions + # (trim-unsafe reflection, missing exports, broken C-ABI marshaling). + # Windows-only for now; the smoke test only exercises Windows paths. + csharp-aot: + name: "C# NativeAOT smoke test (windows-2022)" + runs-on: windows-2022 + # if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + if: false + timeout-minutes: 30 + steps: + - name: Checkout ADBC + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: recursive + - name: Install .NET 10 + uses: actions/setup-dotnet@v5 + with: + # NativeAOT for our producer requires net10. Using 10.0.x selects + # the latest available SDK; preview tags may be needed until GA. + dotnet-version: '10.0.x' + - name: Setup MSVC (for NativeAOT linker) + # Third-party action; activates vcvars64 so ilc's downstream + # link.exe step can find link.exe/lib.exe and the Windows SDK. + # The existing workflow only uses first-party actions, so flag + # this for review before merging. + uses: ilammy/msvc-dev-cmd@v1 + - name: Publish NativeAOT driver + shell: bash + run: | + dotnet publish \ + csharp/src/Drivers/Apache/Apache.Arrow.Adbc.Drivers.Apache.Native/Apache.Arrow.Adbc.Drivers.Apache.Native.csproj \ + -c Release -r win-x64 + - name: Install Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install Python dependencies + shell: bash + run: python -m pip install adbc_driver_manager + - name: Run Python smoke test + shell: bash + run: python csharp/src/Drivers/Apache/Apache.Arrow.Adbc.Drivers.Apache.Native/smoke_test.py diff --git a/csharp/src/Apache.Arrow.Adbc/C/CAdbcDriverExporter.cs b/csharp/src/Apache.Arrow.Adbc/C/CAdbcDriverExporter.cs index 9b6a0c7147..0aa1d46cea 100644 --- a/csharp/src/Apache.Arrow.Adbc/C/CAdbcDriverExporter.cs +++ b/csharp/src/Apache.Arrow.Adbc/C/CAdbcDriverExporter.cs @@ -86,8 +86,8 @@ public class CAdbcDriverExporter private static unsafe IntPtr StatementExecutePartitionsPtr = NativeDelegate.AsNativePointer(ExecuteStatementPartitions); private static unsafe IntPtr StatementExecuteSchemaPtr = NativeDelegate.AsNativePointer(ExecuteStatementSchema); private static unsafe IntPtr StatementNewPtr = NativeDelegate.AsNativePointer(NewStatement); - private static unsafe IntPtr StatementReleasePtr = NativeDelegate.AsNativePointer(ReleaseStatement); - private static unsafe IntPtr StatementPreparePtr = NativeDelegate.AsNativePointer(PrepareStatement); + private static unsafe IntPtr StatementReleasePtr = NativeDelegate.AsNativePointer(ReleaseStatement); + private static unsafe IntPtr StatementPreparePtr = NativeDelegate.AsNativePointer(PrepareStatement); private static unsafe IntPtr StatementSetSqlQueryPtr = NativeDelegate.AsNativePointer(SetStatementSqlQuery); private static unsafe IntPtr StatementSetSubstraitPlanPtr = NativeDelegate.AsNativePointer(SetStatementSubstraitPlan); private static unsafe IntPtr StatementGetParameterSchemaPtr = NativeDelegate.AsNativePointer(GetStatementParameterSchema); diff --git a/csharp/src/Apache.Arrow.Adbc/C/CAdbcDriverImporter.Defaults.cs b/csharp/src/Apache.Arrow.Adbc/C/CAdbcDriverImporter.Defaults.cs index ed2377ba2f..9408d299b4 100644 --- a/csharp/src/Apache.Arrow.Adbc/C/CAdbcDriverImporter.Defaults.cs +++ b/csharp/src/Apache.Arrow.Adbc/C/CAdbcDriverImporter.Defaults.cs @@ -169,7 +169,7 @@ private static unsafe AdbcStatusCode StatementGetParameterSchemaDefaultImpl(CAdb } #if !NET5_0_OR_GREATER - private static unsafe IntPtr StatementPrepareDefault = NativeDelegate.AsNativePointer(StatementPrepareDefaultImpl); + private static unsafe IntPtr StatementPrepareDefault = NativeDelegate.AsNativePointer(StatementPrepareDefaultImpl); #else private static unsafe delegate* unmanaged StatementPrepareDefault => &StatementPrepareDefaultImpl; [UnmanagedCallersOnly] diff --git a/csharp/src/Apache.Arrow.Adbc/C/CAdbcDriverImporter.cs b/csharp/src/Apache.Arrow.Adbc/C/CAdbcDriverImporter.cs index af66aac3be..040399f8d3 100644 --- a/csharp/src/Apache.Arrow.Adbc/C/CAdbcDriverImporter.cs +++ b/csharp/src/Apache.Arrow.Adbc/C/CAdbcDriverImporter.cs @@ -79,27 +79,9 @@ public static AdbcDriver Load(string file, bool canUnload, string? entryPoint = } AdbcDriverInit init = Marshal.GetDelegateForFunctionPointer(export); - CAdbcDriver driver = new CAdbcDriver(); - int version; - using (CallHelper caller = new CallHelper()) - { - try - { - caller.Call(init, AdbcVersion.Version_1_1_0, ref driver); - version = AdbcVersion.Version_1_1_0; - } - catch (AdbcException e) when (e.Status == AdbcStatusCode.NotImplemented) - { - caller.Call(init, AdbcVersion.Version_1_0_0, ref driver); - version = AdbcVersion.Version_1_0_0; - } - - ValidateDriver(ref driver, version); - - ImportedAdbcDriver result = new ImportedAdbcDriver(library, driver, version, canUnload); - library = IntPtr.Zero; - return result; - } + ImportedAdbcDriver result = LoadFromInit(init, library, canUnload); + library = IntPtr.Zero; + return result; } finally { @@ -107,6 +89,40 @@ public static AdbcDriver Load(string file, bool canUnload, string? entryPoint = } } + /// + /// Loads an from an in-process delegate that acts as the driver's + /// AdbcDriverInit entry point. Used for round-tripping + /// through the importer in tests without producing a native library. + /// + internal static AdbcDriver Load(AdbcDriverInit init) + { + if (init == null) { throw new ArgumentNullException(nameof(init)); } + return LoadFromInit(init, IntPtr.Zero, canUnload: false); + } + + private static ImportedAdbcDriver LoadFromInit(AdbcDriverInit init, IntPtr library, bool canUnload) + { + CAdbcDriver driver = new CAdbcDriver(); + int version; + using (CallHelper caller = new CallHelper()) + { + try + { + caller.Call(init, AdbcVersion.Version_1_1_0, ref driver); + version = AdbcVersion.Version_1_1_0; + } + catch (AdbcException e) when (e.Status == AdbcStatusCode.NotImplemented) + { + caller.Call(init, AdbcVersion.Version_1_0_0, ref driver); + version = AdbcVersion.Version_1_0_0; + } + + ValidateDriver(ref driver, version); + + return new ImportedAdbcDriver(library, driver, version, canUnload); + } + } + private static unsafe void ValidateDriver(ref CAdbcDriver driver, int version) { #if NET5_0_OR_GREATER @@ -1036,7 +1052,7 @@ public unsafe override void Prepare() #if NET5_0_OR_GREATER Driver.StatementPrepare #else - Marshal.GetDelegateForFunctionPointer(Driver.StatementPrepare) + Marshal.GetDelegateForFunctionPointer(Driver.StatementPrepare) #endif (statement, &caller._error)); } @@ -1412,7 +1428,7 @@ public unsafe void Call(IntPtr fn, ref CAdbcStatement nativeStatement) fixed (CAdbcStatement* stmt = &nativeStatement) fixed (CAdbcError* e = &_error) { - TranslateCode(Marshal.GetDelegateForFunctionPointer(fn)(stmt, e)); + TranslateCode(Marshal.GetDelegateForFunctionPointer(fn)(stmt, e)); } } #endif diff --git a/csharp/src/Apache.Arrow.Adbc/C/Delegates.cs b/csharp/src/Apache.Arrow.Adbc/C/Delegates.cs index 9469ee58ea..feef7f4a70 100644 --- a/csharp/src/Apache.Arrow.Adbc/C/Delegates.cs +++ b/csharp/src/Apache.Arrow.Adbc/C/Delegates.cs @@ -66,19 +66,18 @@ namespace Apache.Arrow.Adbc.C internal unsafe delegate AdbcStatusCode StatementExecuteQuery(CAdbcStatement* statement, CArrowArrayStream* stream, long* rows, CAdbcError* error); internal unsafe delegate AdbcStatusCode StatementExecutePartitions(CAdbcStatement* statement, CArrowSchema* schema, CAdbcPartitions* partitions, long* rows, CAdbcError* error); internal unsafe delegate AdbcStatusCode StatementExecuteSchema(CAdbcStatement* statement, CArrowSchema* stream, CAdbcError* error); + internal unsafe delegate AdbcStatusCode StatementFn(CAdbcStatement* statement, CAdbcError* error); internal unsafe delegate AdbcStatusCode StatementGetParameterSchema(CAdbcStatement* statement, CArrowSchema* schema, CAdbcError* error); internal unsafe delegate AdbcStatusCode StatementGetOption(CAdbcStatement* statement, byte* name, byte* value, nint* length, CAdbcError* error); internal unsafe delegate AdbcStatusCode StatementGetOptionBytes(CAdbcStatement* statement, byte* name, byte* value, nint* length, CAdbcError* error); internal unsafe delegate AdbcStatusCode StatementGetOptionDouble(CAdbcStatement* statement, byte* name, double* value, CAdbcError* error); internal unsafe delegate AdbcStatusCode StatementGetOptionInt(CAdbcStatement* statement, byte* name, long* value, CAdbcError* error); internal unsafe delegate AdbcStatusCode StatementNew(CAdbcConnection* connection, CAdbcStatement* statement, CAdbcError* error); - internal unsafe delegate AdbcStatusCode StatementPrepare(CAdbcStatement* statement, CAdbcError* error); internal unsafe delegate AdbcStatusCode StatementSetOption(CAdbcStatement* statement, byte* name, byte* value, CAdbcError* error); internal unsafe delegate AdbcStatusCode StatementSetOptionBytes(CAdbcStatement* statement, byte* name, byte* value, nint length, CAdbcError* error); internal unsafe delegate AdbcStatusCode StatementSetOptionDouble(CAdbcStatement* statement, byte* name, double value, CAdbcError* error); internal unsafe delegate AdbcStatusCode StatementSetOptionInt(CAdbcStatement* statement, byte* name, long value, CAdbcError* error); internal unsafe delegate AdbcStatusCode StatementSetSqlQuery(CAdbcStatement* statement, byte* text, CAdbcError* error); internal unsafe delegate AdbcStatusCode StatementSetSubstraitPlan(CAdbcStatement* statement, byte* plan, int length, CAdbcError* error); - internal unsafe delegate AdbcStatusCode StatementRelease(CAdbcStatement* statement, CAdbcError* error); #endif } diff --git a/csharp/src/Apache.Arrow.Adbc/Tracing/ActivityTrace.cs b/csharp/src/Apache.Arrow.Adbc/Tracing/ActivityTrace.cs index 422028094b..273fa930cd 100644 --- a/csharp/src/Apache.Arrow.Adbc/Tracing/ActivityTrace.cs +++ b/csharp/src/Apache.Arrow.Adbc/Tracing/ActivityTrace.cs @@ -274,7 +274,7 @@ public static string GetAssemblyVersion(Type type) if (versionAttr?.InformationalVersion != null) return versionAttr.InformationalVersion; #if NET8_0_OR_GREATER - if (RuntimeFeature.IsDynamicCodeSupported && type.Assembly.Location != null) + if (RuntimeFeature.IsDynamicCodeSupported && !string.IsNullOrEmpty(type.Assembly.Location)) { var fileVersion = FileVersionInfo.GetVersionInfo(type.Assembly.Location).ProductVersion; if (fileVersion != null) return fileVersion; diff --git a/csharp/src/Telemetry/Traces/Listeners/FileListener/OtelAttributesConverter.cs b/csharp/src/Telemetry/Traces/Listeners/FileListener/OtelAttributesConverter.cs index 4bc9b4fa6c..54ff697394 100644 --- a/csharp/src/Telemetry/Traces/Listeners/FileListener/OtelAttributesConverter.cs +++ b/csharp/src/Telemetry/Traces/Listeners/FileListener/OtelAttributesConverter.cs @@ -95,6 +95,11 @@ public static void WriteValue(Utf8JsonWriter writer, object? value) case double[] ds: WriteArray(writer, ds); return; + // Non-OTel types: invariant-culture string. This keeps the output + // portable across locales for things like DateTime or custom structs. + case IFormattable formattable: + writer.WriteStringValue(formattable.ToString(null, CultureInfo.InvariantCulture)); + return; // Generic enumerable fallback (covers ImmutableArray, List, etc.) — // each element is dispatched recursively. case IEnumerable enumerable: @@ -105,11 +110,6 @@ public static void WriteValue(Utf8JsonWriter writer, object? value) } writer.WriteEndArray(); return; - // Non-OTel types: invariant-culture string. This keeps the output - // portable across locales for things like DateTime or custom structs. - case IFormattable formattable: - writer.WriteStringValue(formattable.ToString(null, CultureInfo.InvariantCulture)); - return; default: writer.WriteStringValue(value.ToString()); return; diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Testing.csproj b/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Testing.csproj index ce19532df7..05f9896d7f 100644 --- a/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Testing.csproj +++ b/csharp/test/Apache.Arrow.Adbc.Tests/Apache.Arrow.Adbc.Testing.csproj @@ -5,6 +5,7 @@ net8.0 true true + true $([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant()) diff --git a/csharp/test/Apache.Arrow.Adbc.Tests/ExportedDriverRoundTripTests.cs b/csharp/test/Apache.Arrow.Adbc.Tests/ExportedDriverRoundTripTests.cs new file mode 100644 index 0000000000..071e8805ea --- /dev/null +++ b/csharp/test/Apache.Arrow.Adbc.Tests/ExportedDriverRoundTripTests.cs @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Apache.Arrow.Adbc.C; +using Apache.Arrow.Ipc; +using Apache.Arrow.Types; +using Xunit; + +namespace Apache.Arrow.Adbc.Tests +{ + /// + /// Runs a managed fixture through + /// to populate a CAdbcDriver struct, + /// then loads that struct via as if it + /// came from a native library. Exercises the ADBC 1.0 C-ABI marshaling + /// without producing or loading a native DLL. + /// + public class ExportedDriverRoundTripTests + { + [Fact] + public async Task RoundTripSimpleQuery() + { + var fixture = new FixtureDriver(); + + using AdbcDriver imported = CAdbcDriverImporter.Load(CreateAdapter(fixture)); + + using AdbcDatabase db = imported.Open(new Dictionary { { "uri", "ignored" } }); + using AdbcConnection conn = db.Connect(null); + using AdbcStatement stmt = conn.CreateStatement(); + + stmt.SqlQuery = "SELECT 42"; + QueryResult result = stmt.ExecuteQuery(); + + using IArrowArrayStream stream = result.Stream!; + Assert.NotNull(stream); + + Schema schema = stream.Schema; + Assert.Single(schema.FieldsList); + Assert.Equal(ArrowTypeId.Int32, schema.FieldsList[0].DataType.TypeId); + Assert.Equal("answer", schema.FieldsList[0].Name); + + RecordBatch? batch = await stream.ReadNextRecordBatchAsync(); + Assert.NotNull(batch); + Assert.Equal(1, batch!.Length); + Int32Array column = Assert.IsType(batch.Column(0)); + Assert.Equal(42, column.Values[0]); + + Assert.Null(await stream.ReadNextRecordBatchAsync()); + } + + [Fact] + public void SqlQueryIsMarshaledToProducer() + { + var fixture = new FixtureDriver(); + + using AdbcDriver imported = CAdbcDriverImporter.Load(CreateAdapter(fixture)); + using AdbcDatabase db = imported.Open(new Dictionary()); + using AdbcConnection conn = db.Connect(null); + using AdbcStatement stmt = conn.CreateStatement(); + + const string query = "SELECT 'hello', 'world'"; + stmt.SqlQuery = query; + stmt.ExecuteQuery(); + + Assert.Equal(query, fixture.LastStatement!.ReceivedQuery); + } + + [Fact] + public void OpenParametersAreMarshaledToProducer() + { + var fixture = new FixtureDriver(); + + using AdbcDriver imported = CAdbcDriverImporter.Load(CreateAdapter(fixture)); + using AdbcDatabase db = imported.Open(new Dictionary + { + { "first", "1" }, + { "second", "2" }, + }); + + Assert.Equal("1", fixture.LastDatabase!.Options["first"]); + Assert.Equal("2", fixture.LastDatabase!.Options["second"]); + } + + [Fact] + public void ProducerExceptionPropagatesAsAdbcException() + { + var fixture = new FixtureDriver { ThrowOnExecute = new InvalidOperationException("boom") }; + + using AdbcDriver imported = CAdbcDriverImporter.Load(CreateAdapter(fixture)); + using AdbcDatabase db = imported.Open(new Dictionary()); + using AdbcConnection conn = db.Connect(null); + using AdbcStatement stmt = conn.CreateStatement(); + stmt.SqlQuery = "SELECT 1"; + + AdbcException ex = Assert.ThrowsAny(() => stmt.ExecuteQuery()); + Assert.Contains("boom", ex.Message); + } + + private static AdbcDriverInit CreateAdapter(AdbcDriver driver) + { + return (int version, ref CAdbcDriver nativeDriver, ref CAdbcError error) => + { + unsafe + { + fixed (CAdbcDriver* dp = &nativeDriver) + fixed (CAdbcError* ep = &error) + { + return CAdbcDriverExporter.AdbcDriverInit(version, dp, ep, driver); + } + } + }; + } + + private sealed class FixtureDriver : AdbcDriver + { + public FixtureDatabase? LastDatabase { get; private set; } + public FixtureStatement? LastStatement { get; private set; } + public Exception? ThrowOnExecute { get; set; } + + public override AdbcDatabase Open(IReadOnlyDictionary parameters) + { + var db = new FixtureDatabase(this, parameters); + LastDatabase = db; + return db; + } + + internal void RecordStatement(FixtureStatement stmt) => LastStatement = stmt; + } + + private sealed class FixtureDatabase : AdbcDatabase + { + private readonly FixtureDriver _driver; + public Dictionary Options { get; } + + public FixtureDatabase(FixtureDriver driver, IReadOnlyDictionary parameters) + { + _driver = driver; +#if NET6_0_OR_GREATER + Options = new Dictionary(parameters); +#else + Options = new Dictionary(parameters.Count); + foreach (KeyValuePair pair in parameters) + { + Options.Add(pair.Key, pair.Value); + } +#endif + } + + public override void SetOption(string key, string value) => Options[key] = value; + + public override AdbcConnection Connect(IReadOnlyDictionary? options) + => new FixtureConnection(_driver); + } + + private sealed class FixtureConnection : AdbcConnection + { + private readonly FixtureDriver _driver; + + public FixtureConnection(FixtureDriver driver) { _driver = driver; } + + public override AdbcStatement CreateStatement() + { + var stmt = new FixtureStatement(_driver); + _driver.RecordStatement(stmt); + return stmt; + } + + public override IArrowArrayStream GetObjects( + GetObjectsDepth depth, string? catalogPattern, string? dbSchemaPattern, + string? tableNamePattern, IReadOnlyList? tableTypes, string? columnNamePattern) + => throw AdbcException.NotImplemented("fixture does not support GetObjects"); + + public override Schema GetTableSchema(string? catalog, string? dbSchema, string tableName) + => throw AdbcException.NotImplemented("fixture does not support GetTableSchema"); + + public override IArrowArrayStream GetTableTypes() + => throw AdbcException.NotImplemented("fixture does not support GetTableTypes"); + } + + private sealed class FixtureStatement : AdbcStatement + { + private readonly FixtureDriver _driver; + private string? _sqlQuery; + + public FixtureStatement(FixtureDriver driver) { _driver = driver; } + + public string? ReceivedQuery => _sqlQuery; + + public override string? SqlQuery + { + get => _sqlQuery; + set => _sqlQuery = value; + } + + public override QueryResult ExecuteQuery() + { + if (_driver.ThrowOnExecute != null) { throw _driver.ThrowOnExecute; } + + var schema = new Schema.Builder() + .Field(f => f.Name("answer").DataType(Int32Type.Default).Nullable(false)) + .Build(); + var column = new Int32Array.Builder().Append(42).Build(); + var batch = new RecordBatch(schema, new IArrowArray[] { column }, 1); + return new QueryResult(1, new SingleBatchStream(schema, batch)); + } + + public override UpdateResult ExecuteUpdate() + => throw AdbcException.NotImplemented("fixture does not support ExecuteUpdate"); + } + + private sealed class SingleBatchStream : IArrowArrayStream + { + private readonly Schema _schema; + private RecordBatch? _batch; + + public SingleBatchStream(Schema schema, RecordBatch batch) + { + _schema = schema; + _batch = batch; + } + + public Schema Schema => _schema; + + public ValueTask ReadNextRecordBatchAsync(CancellationToken cancellationToken = default) + { + RecordBatch? result = _batch; + _batch = null; + return new ValueTask(result); + } + + public void Dispose() { _batch?.Dispose(); _batch = null; } + } + } +} From 5f37a1783b53a43ca10fb2f84562c45176380ba2 Mon Sep 17 00:00:00 2001 From: Curt Hagenlocher Date: Sun, 19 Apr 2026 06:36:38 -0700 Subject: [PATCH 3/3] Pin currently-unused MSVC task --- .github/workflows/csharp.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml index ffd740e33f..b5f9e0545d 100644 --- a/.github/workflows/csharp.yml +++ b/.github/workflows/csharp.yml @@ -98,8 +98,8 @@ jobs: # Third-party action; activates vcvars64 so ilc's downstream # link.exe step can find link.exe/lib.exe and the Windows SDK. # The existing workflow only uses first-party actions, so flag - # this for review before merging. - uses: ilammy/msvc-dev-cmd@v1 + # this for review before enabling. + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1 - name: Publish NativeAOT driver shell: bash run: |